{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=45.08: 100%|█| 20/20 [01:02<00:00,  3.11s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ6klEQVR4nO3dd3iUVeL28e+U9J5AEgIBQu9FghhA0TUKgijKropRcVdlVVDRFZFXcW0Lih1kYXX3Z9nFsq6ACoIgVaSXUCMdEkoSIL0nM8/7R8jgSGghyUyS+3Ndc13M85yZ5xxYk3tPNRmGYSAiIiLSgJldXQERERERV1MgEhERkQZPgUhEREQaPAUiERERafAUiERERKTBUyASERGRBk+BSERERBo8q6srUBfY7XaOHTtGQEAAJpPJ1dURERGRi2AYBrm5uURFRWE2n78PSIHoIhw7dozo6GhXV0NERESqICUlhWbNmp23jALRRQgICADK/0IDAwNdXBsRERG5GDk5OURHRzt+j5+PSwPRypUreeONN9i0aRPHjx9nzpw5DBs2DIDS0lKef/55vv/+ew4cOEBQUBDx8fG89tprREVFOb4jIyODxx57jO+++w6z2czw4cN577338Pf3d5TZtm0bo0ePZsOGDTRu3JjHHnuMZ5555qLrWTFMFhgYqEAkIiJSx1zMdBeXTqrOz8+ne/fuTJ8+/ax7BQUFbN68mYkTJ7J582Zmz57N7t27ueWWW5zKJSQksHPnThYvXsy8efNYuXIlo0aNctzPycnhxhtvpEWLFmzatIk33niDF198kQ8++KDG2yciIiJ1g8ldDnc1mUxOPUSV2bBhA1deeSWHDx+mefPmJCUl0alTJzZs2EBsbCwACxcuZPDgwRw5coSoqChmzJjBc889R2pqKp6engA8++yzzJ07l19++eWi6paTk0NQUBDZ2dnqIRIREakjLuX3d51adp+dnY3JZCI4OBiANWvWEBwc7AhDAPHx8ZjNZtatW+coc8011zjCEMDAgQPZvXs3mZmZlT6nuLiYnJwcp5eIiIjUX3VmUnVRURHjx49nxIgRjpSXmppKeHi4Uzmr1UpoaCipqamOMjExMU5lIiIiHPdCQkLOetbkyZN56aWXaqIZIiLipmw2G6Wlpa6uhlwiT0/PCy6pvxh1IhCVlpZyxx13YBgGM2bMqPHnTZgwgaeeesrxvmKWuoiI1D+GYZCamkpWVparqyJVYDabiYmJcRoJqgq3D0QVYejw4cMsXbrUaQwwMjKS9PR0p/JlZWVkZGQQGRnpKJOWluZUpuJ9RZnf8vLywsvLqzqbISIibqoiDIWHh+Pr66sNeOuQio2Tjx8/TvPmzS/r386tA1FFGNq7dy/Lli0jLCzM6X5cXBxZWVls2rSJXr16AbB06VLsdjt9+vRxlHnuuecoLS3Fw8MDgMWLF9O+fftKh8tERKThsNlsjjD0298xUjc0btyYY8eOUVZW5vg9XxUunVSdl5dHYmIiiYmJABw8eJDExESSk5MpLS3l97//PRs3bmTWrFnYbDZSU1NJTU2lpKQEgI4dOzJo0CAeeugh1q9fz88//8yYMWO46667HHsV3X333Xh6evLAAw+wc+dOvvzyS9577z2nITEREWmYKuYM+fr6urgmUlUVQ2U2m+2yvsely+6XL1/Oddddd9b1kSNH8uKLL541GbrCsmXLuPbaa4HyjRnHjBnjtDHj1KlTz7kxY6NGjXjssccYP378RddTy+5FROqnoqIiDh48SExMDN7e3q6ujlTB+f4NL+X3t0uHzK699lrOl8cuJquFhoby2WefnbdMt27d+Omnny65fiIiItIw1Kl9iERERERqggKRiIhIA9eyZUveffddl3+HK7n1KrP6zjAMMgtKycgvpk34hU/iFRERgfIpJz169Ki2ALJhwwb8/Pyq5bvqKgUiFzpwMp/r31qBn6eFHS8N1N4XIiJSbQzDwGazYbVe+Fd948aNa6FG7k1DZi4UFeQDQH6JjdziMhfXRkREoDxIFJSU1frrYhd933///axYsYL33nsPk8mEyWTi0KFDLF++HJPJxIIFC+jVqxdeXl6sWrWK/fv3c+uttxIREYG/vz+9e/fmxx9/dPrO3w53mUwm/vnPf3Lbbbfh6+tL27Zt+fbbby/p7zE5OZlbb70Vf39/AgMDueOOO5w2St66dSvXXXcdAQEBBAYG0qtXLzZu3AjA4cOHGTp0KCEhIfj5+dG5c2e+//77S3r+pVIPkQv5eFoI9vUgq6CU41lFBEZWfUMpERGpHoWlNjq98EOtP3fXywPx9bzwr+X33nuPPXv20KVLF15++WWgvIfn0KFDADz77LO8+eabtGrVipCQEFJSUhg8eDB/+9vf8PLy4tNPP2Xo0KHs3r2b5s2bn/M5L730ElOmTOGNN95g2rRpJCQkcPjwYUJDQy9YR7vd7ghDK1asoKysjNGjR3PnnXeyfPlyABISEujZsyczZszAYrGQmJjo2Fhx9OjRlJSUsHLlSvz8/Ni1a5fTdjo1QYHIxSIDvcsDUXYh7SM1j0hERM4vKCgIT09PfH19Kz2C6uWXX+aGG25wvA8NDaV79+6O96+88gpz5szh22+/ZcyYMed8zv3338+IESMAmDRpElOnTmX9+vUMGjTognVcsmQJ27dv5+DBg46zQD/99FM6d+7Mhg0b6N27N8nJyYwbN44OHToA0LZtW8fnk5OTGT58OF27dgWgVatWF3zm5VIgcrGoYB9+Sc3leHaRq6siIiKAj4eFXS8PdMlzq0NsbKzT+7y8PF588UXmz5/P8ePHKSsro7CwkOTk5PN+T7du3Rx/9vPzIzAw8KzzQ88lKSmJ6Ohop4PRO3XqRHBwMElJSfTu3ZunnnqKBx98kH//+9/Ex8fzhz/8gdatWwPw+OOP88gjj7Bo0SLi4+MZPny4U31qguYQuVhkUPmumgpEIiLuwWQy4etprfVXdS2s+e1qsaeffpo5c+YwadIkfvrpJxITE+natavjGKxz+e25YCaTCbvdXi11BHjxxRfZuXMnQ4YMYenSpXTq1Ik5c+YA8OCDD3LgwAHuvfdetm/fTmxsLNOmTau2Z1dGgcjFoioCUVahi2siIiJ1haen50Wf3fXzzz9z//33c9ttt9G1a1ciIyMd841qSseOHUlJSSElJcVxbdeuXWRlZdGpUyfHtXbt2vHkk0+yaNEibr/9dj766CPHvejoaB5++GFmz57NX/7yFz788MMarbMCkYtFnl5plpqjHiIREbk4LVu2ZN26dRw6dIiTJ0+et+embdu2zJ49m8TERLZu3crdd99drT09lYmPj6dr164kJCSwefNm1q9fz3333ceAAQOIjY2lsLCQMWPGsHz5cg4fPszPP//Mhg0b6NixIwBjx47lhx9+4ODBg2zevJlly5Y57tUUBSIXq+ghOqYeIhERuUhPP/00FouFTp060bhx4/POB3r77bcJCQmhb9++DB06lIEDB3LFFVfUaP1MJhPffPMNISEhXHPNNcTHx9OqVSu+/PJLACwWC6dOneK+++6jXbt23HHHHdx000289NJLQPnJ9aNHj6Zjx44MGjSIdu3a8fe//71m6+zK0+7ripo87f7AiTx+99YKfD0t7NTmjCIitUqn3dd91XXavXqIXKzJ6SGzghIbOUXanFFERMQVFIhczMfTQohv+Uz+49kaNhMREXEFBSI3UDGxWkvvRUREXEOByA2cWXqvQCQi4gqaTlt3Vde/nQKRG6jYnDFVQ2YiIrWqYvPBgoICF9dEqqpig0mL5fJ2+tbRHW4gKrh8yOyYhsxERGqVxWIhODjYcSSFr6+vVvvWIXa7nRMnTuDr64vVenmRRoHIDUQGVvQQKRCJiNS2igNSL/acLnEvZrOZ5s2bX3aQVSByA02CT2/OqCEzEZFaZzKZaNKkCeHh4ZSWlrq6OnKJPD09MZsvfwaQApEbqNiL6HhWEYZhqLtWRMQFLBbLZc9DkbpLk6rdQJPTk6oLS23kFGpzRhERkdqmQOQGvD0shPp5Aho2ExERcQUFIjehidUiIiKuo0DkJqI0sVpERMRlFIjcxJnNGdVDJCIiUtsUiNxExUqzYzq+Q0REpNYpELmJipVmqTkaMhMREaltCkRu4td7EYmIiEjtUiByExWTqo9rDpGIiEitUyByExX7EBWW2igssbm4NiIiIg2LApGb8PeyYjWXH9mRVVji4tqIiIg0LApEbsJkMhHsW95LlJmvwwVFRERqkwKRGwn29QAgq0A9RCIiIrVJgciNhJwORJkF6iESERGpTQpEbqRiyExziERERGqXApEbCXEMmamHSEREpDYpELmRM5Oq1UMkIiJSmxSI3Eiw5hCJiIi4hAKRGwk53UOUrTlEIiIitUqByI1olZmIiIhrKBC5kSCf03OItA+RiIhIrVIgciMhflplJiIi4goKRG6kYg5RVkEJdrvh4tqIiIg0HApEbiTIp7yHyG5AbnGZi2sjIiLScCgQuRFvDws+HhZA55mJiIjUJgUiN6OVZiIiIrVPgcjNBP9qHpGIiIjUDpcGopUrVzJ06FCioqIwmUzMnTvX6b5hGLzwwgs0adIEHx8f4uPj2bt3r1OZjIwMEhISCAwMJDg4mAceeIC8vDynMtu2bePqq6/G29ub6OhopkyZUtNNqzKtNBMREal9Lg1E+fn5dO/enenTp1d6f8qUKUydOpWZM2eybt06/Pz8GDhwIEVFRY4yCQkJ7Ny5k8WLFzNv3jxWrlzJqFGjHPdzcnK48cYbadGiBZs2beKNN97gxRdf5IMPPqjx9lVFsPYiEhERqXVWVz78pptu4qabbqr0nmEYvPvuuzz//PPceuutAHz66adEREQwd+5c7rrrLpKSkli4cCEbNmwgNjYWgGnTpjF48GDefPNNoqKimDVrFiUlJfzf//0fnp6edO7cmcTERN5++22n4OQudJ6ZiIhI7XPbOUQHDx4kNTWV+Ph4x7WgoCD69OnDmjVrAFizZg3BwcGOMAQQHx+P2Wxm3bp1jjLXXHMNnp6ejjIDBw5k9+7dZGZmVvrs4uJicnJynF61xXGemXqIREREao3bBqLU1FQAIiIinK5HREQ47qWmphIeHu5032q1Ehoa6lSmsu/49TN+a/LkyQQFBTle0dHRl9+gi6QeIhERkdrntoHIlSZMmEB2drbjlZKSUmvPrlhlpjlEIiIitcdtA1FkZCQAaWlpTtfT0tIc9yIjI0lPT3e6X1ZWRkZGhlOZyr7j18/4LS8vLwIDA51etaViHyKtMhMREak9bhuIYmJiiIyMZMmSJY5rOTk5rFu3jri4OADi4uLIyspi06ZNjjJLly7FbrfTp08fR5mVK1dSWnomYCxevJj27dsTEhJSS625eI59iArVQyQiIlJbXBqI8vLySExMJDExESifSJ2YmEhycjImk4mxY8fy6quv8u2337J9+3buu+8+oqKiGDZsGAAdO3Zk0KBBPPTQQ6xfv56ff/6ZMWPGcNdddxEVFQXA3XffjaenJw888AA7d+7kyy+/5L333uOpp55yUavPz9FDlK8eIhERkdri0mX3Gzdu5LrrrnO8rwgpI0eO5OOPP+aZZ54hPz+fUaNGkZWVRf/+/Vm4cCHe3t6Oz8yaNYsxY8Zw/fXXYzabGT58OFOnTnXcDwoKYtGiRYwePZpevXrRqFEjXnjhBbdccg9neohyi8sotdnxsLhtJ56IiEi9YTIMw3B1JdxdTk4OQUFBZGdn1/h8IpvdoM1z32MYsOG5eBoHeNXo80REROqrS/n9re4HN2Mxmwj0Lh82y9Y8IhERkVqhQOSGdOK9iIhI7VIgckNBFXsR5auHSEREpDYoELkh7UUkIiJSuxSI3FCI9iISERGpVQpEbkjnmYmIiNQuBSI3FOxzuodI55mJiIjUCgUiNxTid7qHSLtVi4iI1AoFIjekE+9FRERqlwKRG6pYZZZdqB4iERGR2qBA5IYq5hCph0hERKR2KBC5oV+vMtNRcyIiIjVPgcgNhfmX9xCVlNnJL7G5uDYiIiL1nwKRG/L1tOLtUf5Pk5GnYTMREZGapkDkpkJPrzQ7lV/s4pqIiIjUfwpEbir09LBZhg54FRERqXEKRG4q1M8LUCASERGpDQpEbirMTz1EIiIitUWByE2FKhCJiIjUGgUiN1URiE4pEImIiNQ4BSI3pSEzERGR2qNA5KbUQyQiIlJ7FIjc1Jk5RNqHSEREpKYpELkpRyDSTtUiIiI1ToHITYWd3ocov8RGUanOMxMREalJCkRuKtDHitVsAiCzQL1EIiIiNUmByE2ZTCZCKiZWa9hMRESkRikQuTEtvRcREakdCkRuTLtVi4iI1A4FIjemvYhERERqhwKRG9NeRCIiIrVDgciNachMRESkdigQubEwrTITERGpFQpEbiz09OaM2odIRESkZikQuTFNqhYREakdCkRuLMxfc4hERERqgwKRG6voIcoqKKXMZndxbUREROovBSI3Fuzj4fhzZkGpC2siIiJSvykQuTGrxUywb3ko0rCZiIhIzVEgcnNnJlZrc0YREZGaokDk5nTAq4iISM1TIHJz2q1aRESk5ikQubmKzRkViERERGqOApGb05CZiIhIzVMgcnParVpERKTmKRC5OcccIh3wKiIiUmMUiNycJlWLiIjUPAUiN6chMxERkZqnQOTmGvmXrzLLLCjBbjdcXBsREZH6ya0Dkc1mY+LEicTExODj40Pr1q155ZVXMIwzwcAwDF544QWaNGmCj48P8fHx7N271+l7MjIySEhIIDAwkODgYB544AHy8vJquzlVUnHivc1ukFmgXiIREZGa4NaB6PXXX2fGjBm8//77JCUl8frrrzNlyhSmTZvmKDNlyhSmTp3KzJkzWbduHX5+fgwcOJCioiJHmYSEBHbu3MnixYuZN28eK1euZNSoUa5o0iXzsJgJOX2e2Yk8Hd8hIiJSE6yursD5rF69mltvvZUhQ4YA0LJlSz7//HPWr18PlPcOvfvuuzz//PPceuutAHz66adEREQwd+5c7rrrLpKSkli4cCEbNmwgNjYWgGnTpjF48GDefPNNoqKiXNO4S9A4wIvMglJO5BbTIdLVtREREal/3LqHqG/fvixZsoQ9e/YAsHXrVlatWsVNN90EwMGDB0lNTSU+Pt7xmaCgIPr06cOaNWsAWLNmDcHBwY4wBBAfH4/ZbGbdunWVPre4uJicnBynlys1DiifR3RSPUQiIiI1wq17iJ599llycnLo0KEDFosFm83G3/72NxISEgBITU0FICIiwulzERERjnupqamEh4c73bdarYSGhjrK/NbkyZN56aWXqrs5VVYxsfpErgKRiIhITXDrHqL//ve/zJo1i88++4zNmzfzySef8Oabb/LJJ5/U6HMnTJhAdna245WSklKjz7uQxv4VPUSaVC0iIlIT3LqHaNy4cTz77LPcddddAHTt2pXDhw8zefJkRo4cSWRk+YSatLQ0mjRp4vhcWloaPXr0ACAyMpL09HSn7y0rKyMjI8Px+d/y8vLCy8urBlpUNY0C1EMkIiJSk9y6h6igoACz2bmKFosFu90OQExMDJGRkSxZssRxPycnh3Xr1hEXFwdAXFwcWVlZbNq0yVFm6dKl2O12+vTpUwutuHxneogUiERERGqCW/cQDR06lL/97W80b96czp07s2XLFt5++23+9Kc/AWAymRg7diyvvvoqbdu2JSYmhokTJxIVFcWwYcMA6NixI4MGDeKhhx5i5syZlJaWMmbMGO666646scIMzkyqVg+RiIhIzXDrQDRt2jQmTpzIo48+Snp6OlFRUfz5z3/mhRdecJR55plnyM/PZ9SoUWRlZdG/f38WLlyIt7e3o8ysWbMYM2YM119/PWazmeHDhzN16lRXNKlKNKlaRESkZpmMX2/7LJXKyckhKCiI7OxsAgMDa/35J3KL6f23HzGZYO+rN2G1uPVIp4iIiFu4lN/f+s1aB4T6eWI2gWHo1HsREZGaoEBUB1jMJkL9Tg+baWK1iIhItVMgqiM0sVpERKTmKBDVEY1On3qvzRlFRESqnwJRHaEeIhERkZqjQFRHNNbSexERkRqjQFRH6MR7ERGRmqNAVEdoyExERKTmKBDVEY10npmIiEiNUSCqIxw9RApEIiIi1U6BqI6omFSdVVBKSZndxbURERGpXxSI6oggHw+sZhMAp/LVSyQiIlKdFIjqCLPZpFPvRUREaogCUR3SKKBit2oFIhERkeqkQFSHaHNGERGRmqFAVIec2ZxR55mJiIhUJwWiOkRziERERGqGAlEdor2IREREaoYCUR2iHiIREZGaoUBUhzjmECkQiYiIVCsFojrE0UOkITMREZFqpUBUh4T6le9DlFtURqlNx3eIiIhUFwWiOiTIxwNT+ekdZBWUurYyIiIi9YgCUR1iMZsI8vEAIKtAexGJiIhUFwWiOibUt3zYLCNfgUhERKS6KBDVMcG+5T1EmRoyExERqTYKRHVMxcTqTA2ZiYiIVBsFojom2FeBSEREpLopENUxjh4izSESERGpNgpEdUyIo4dIc4hERESqiwJRHRNSMalaPUQiIiLVRoGojgnRpGoREZFqp0BUx2jITEREpPopENUxoX4V+xCph0hERKS6KBDVMRXL7rMLSynTAa8iIiLVQoGojgk+fZaZYZSHIhEREbl8CkR1jNViJtDbCmgekYiISHVRIKqDdHyHiIhI9VIgqoMcx3doLyIREZFqoUBUB6mHSEREpHopENVBwRW7VWsOkYiISLVQIKqDQjVkJiIiUq0UiOogHd8hIiJSvaoUiD755BPmz5/veP/MM88QHBxM3759OXz4cLVVTipXcXxHRr6GzERERKpDlQLRpEmT8PHxAWDNmjVMnz6dKVOm0KhRI5588slqraCcreLE+yz1EImIiFQLa1U+lJKSQps2bQCYO3cuw4cPZ9SoUfTr149rr722OusnlagYMstQIBIREakWVeoh8vf359SpUwAsWrSIG264AQBvb28KCwurr3ZSqYohsyytMhMREakWVeohuuGGG3jwwQfp2bMne/bsYfDgwQDs3LmTli1bVmf9pBIhfmeGzOx2A7PZ5OIaiYiI1G1V6iGaPn06cXFxnDhxgq+//pqwsDAANm3axIgRI6q1gnK2ih4iuwE5ReolEhERuVxVCkTBwcG8//77fPPNNwwaNMhx/aWXXuK5556rtsoBHD16lHvuuYewsDB8fHzo2rUrGzdudNw3DIMXXniBJk2a4OPjQ3x8PHv37nX6joyMDBISEggMDCQ4OJgHHniAvLy8aq1nbfKwmAnwKu/cy9BeRCIiIpetSoFo4cKFrFq1yvF++vTp9OjRg7vvvpvMzMxqq1xmZib9+vXDw8ODBQsWsGvXLt566y1CQkIcZaZMmcLUqVOZOXMm69atw8/Pj4EDB1JUVOQok5CQwM6dO1m8eDHz5s1j5cqVjBo1qtrq6Qpn9iJSD5GIiMhlM6qgS5cuxvz58w3DMIxt27YZXl5exoQJE4yrrrrKuP/++6vylZUaP3680b9//3Pet9vtRmRkpPHGG284rmVlZRleXl7G559/bhiGYezatcsAjA0bNjjKLFiwwDCZTMbRo0cvqh7Z2dkGYGRnZ1exJdXvlmk/GS3GzzMW70x1dVVERETc0qX8/q5SD9HBgwfp1KkTAF9//TU333wzkyZNYvr06SxYsKDawtq3335LbGwsf/jDHwgPD6dnz558+OGHTvVITU0lPj7ecS0oKIg+ffqwZs0aoHyfpODgYGJjYx1l4uPjMZvNrFu3rtLnFhcXk5OT4/RyN9qtWkREpPpUKRB5enpSUFAAwI8//siNN94IQGhoaLWGhwMHDjBjxgzatm3LDz/8wCOPPMLjjz/OJ598AkBqaioAERERTp+LiIhw3EtNTSU8PNzpvtVqJTQ01FHmtyZPnkxQUJDjFR0dXW1tqi4VE6sViERERC5flZbd9+/fn6eeeop+/fqxfv16vvzySwD27NlDs2bNqq1ydrud2NhYJk2aBEDPnj3ZsWMHM2fOZOTIkdX2nN+aMGECTz31lON9Tk6O24WiM4FIc4hEREQuV5V6iN5//32sViv/+9//mDFjBk2bNgVgwYIFTqvOLleTJk0cQ3MVOnbsSHJyMgCRkZEApKWlOZVJS0tz3IuMjCQ9Pd3pfllZGRkZGY4yv+Xl5UVgYKDTy91UHN+hE+9FREQuX5V6iJo3b868efPOuv7OO+9cdoV+rV+/fuzevdvp2p49e2jRogUAMTExREZGsmTJEnr06AGU9+asW7eORx55BIC4uDiysrLYtGkTvXr1AmDp0qXY7Xb69OlTrfWtTZpDJCIiUn2qFIgAbDYbc+fOJSkpCYDOnTtzyy23YLFYqq1yTz75JH379mXSpEnccccdrF+/ng8++IAPPvgAAJPJxNixY3n11Vdp27YtMTExTJw4kaioKIYNGwaU9ygNGjSIhx56iJkzZ1JaWsqYMWO46667iIqKqra61jbHkJlOvBcREblsVQpE+/btY/DgwRw9epT27dsD5RORo6OjmT9/Pq1bt66WyvXu3Zs5c+YwYcIEXn75ZWJiYnj33XdJSEhwlHnmmWfIz89n1KhRZGVl0b9/fxYuXIi3t7ejzKxZsxgzZgzXX389ZrOZ4cOHM3Xq1Gqpo6tUHN+hHiIREZHLZzIMw7jUDw0ePBjDMJg1axahoaEAnDp1invuuQez2cz8+fOrvaKulJOTQ1BQENnZ2W4zn+iX1BwGvfsToX6ebJ54g6urIyIi4nYu5fd3lXqIVqxYwdq1ax1hCCAsLIzXXnuNfv36VeUr5RJFBfsA5Ud3FJSU4etZ5dFPERGRBq9Kq8y8vLzIzc0963peXh6enp6XXSm5sEBvD4J8yofNUjIKXVwbERGRuq1Kgejmm29m1KhRrFu3DsMwMAyDtWvX8vDDD3PLLbdUdx3lHJqH+gKQnFHg4pqIiIjUbVUKRFOnTqV169bExcXh7e2Nt7c3ffv2pU2bNrz77rvVXEU5l4pAlKJAJCIiclmqNPEkODiYb775hn379jmW3Xfs2JE2bdpUa+Xk/KLVQyQiIlItLjoQ/fooi8osW7bM8ee333676jWSi6YeIhERkepx0YFoy5YtF1XOZDJVuTJyaaJDy1eaqYdIRETk8lx0IPp1D5C4B0cPUWYBhmEojIqIiFRRlSZVi3uICvbBbIKiUjsn8opdXR0REZE6S4GoDvOwmB0bNGoekYiISNUpENVx2otIRETk8ikQ1XHRIacD0SntVi0iIlJVCkR1XPMw9RCJiIhcLgWiOi76VyvNREREpGoUiOo4bc4oIiJy+RSI6rjokPJVZqk5RRSV2lxcGxERkbpJgaiOC/XzxM/TgmHA0SxNrBYREakKBaI6zmQynZlHpGEzERGRKlEgqgc0j0hEROTyKBDVA9qcUURE5PIoENUD0QpEIiIil0WBqB4400OkSdUiIiJVoUBUD1TsVn34VD52u+Hi2oiIiNQ9CkT1QPNQXzwsJgpKbBzPKXJ1dUREROocBaJ6wMNipmWYHwD70vNcXBsREZG6R4GonmgT7g8oEImIiFSFAlE9cSYQ5bq4JiIiInWPAlE9oR4iERGRqlMgqicqAtHe9DwMQyvNRERELoUCUT3RurE/JhNkFZRyKr/E1dURERGpUxSI6glvDwvRIeX7EWnYTERE5NIoENUjmkckIiJSNQpE9YgCkYiISNUoENUjbRorEImIiFSFAlE90iZCgUhERKQqFIjqkYohs9ScInKKSl1cGxERkbpDgageCfT2IDzAC4D96iUSERG5aApE9UxbDZuJiIhcMgWiesYxsfqEApGIiMjFUiCqZxxL79MUiERERC6WAlE90yY8AIBNyZkknypwcW1ERETqBgWieqZXixDaRfiTVVDK3f9cy7GsQldXSURExO0pENUznlYz/3mgDy3DfDmSWUjCP9eRnlvk6mqJiIi4NQWieig80JtZD11F02AfDp7MZ8ysLa6ukoiIiFtTIKqnmgb7MOvBPljMJtYfyuDQyXxXV0lERMRtKRDVYy0b+dG3dRgA87cfd3FtRERE3JcCUT13c7cmAMzbpkAkIiJyLgpE9dzAzpFYzSaSjuewX5s1ioiIVEqBqJ4L9vWkf9tGAMxXL5GIiEilFIgagCFdy4fNFIhEREQqV6cC0WuvvYbJZGLs2LGOa0VFRYwePZqwsDD8/f0ZPnw4aWlpTp9LTk5myJAh+Pr6Eh4ezrhx4ygrK6vl2rvOjZ0i8bCY2J2Wy960XFdXR0RExO3UmUC0YcMG/vGPf9CtWzen608++STfffcdX331FStWrODYsWPcfvvtjvs2m40hQ4ZQUlLC6tWr+eSTT/j444954YUXarsJLhPk68HVbRsDWm0mIiJSmToRiPLy8khISODDDz8kJCTEcT07O5t//etfvP322/zud7+jV69efPTRR6xevZq1a9cCsGjRInbt2sV//vMfevTowU033cQrr7zC9OnTKSkpqfR5xcXF5OTkOL3quorVZgt3pLq4JiIiIu6nTgSi0aNHM2TIEOLj452ub9q0idLSUqfrHTp0oHnz5qxZswaANWvW0LVrVyIiIhxlBg4cSE5ODjt37qz0eZMnTyYoKMjxio6OroFW1a5+bconVu9Nz6Oo1Obi2oiIiLgXtw9EX3zxBZs3b2by5Mln3UtNTcXT05Pg4GCn6xEREaSmpjrK/DoMVdyvuFeZCRMmkJ2d7XilpKRUQ0tcKzzAixBfD2x2g33pzsvvV+w5wZHMAhfVTERExPXcOhClpKTwxBNPMGvWLLy9vWvtuV5eXgQGBjq96jqTyUSHyPJ2JB0/MwSYmJLFyP9bz21/X01WQeVDiCIiIvWdWweiTZs2kZ6ezhVXXIHVasVqtbJixQqmTp2K1WolIiKCkpISsrKynD6XlpZGZGQkAJGRkWetOqt4X1GmoejQJACAX1LPrDRbs/8UACdyi3n5u10uqZeIiIiruXUguv7669m+fTuJiYmOV2xsLAkJCY4/e3h4sGTJEsdndu/eTXJyMnFxcQDExcWxfft20tPTHWUWL15MYGAgnTp1qvU2uVLH0z1Ev6Se6SHakpzp+PPsLUf5cVfaWZ8TERGp76yursD5BAQE0KVLF6drfn5+hIWFOa4/8MADPPXUU4SGhhIYGMhjjz1GXFwcV111FQA33ngjnTp14t5772XKlCmkpqby/PPPM3r0aLy8vGq9Ta7UsUnFkFkuhmEAsCUlC4A+MaGsO5jB/5uznd4tQwny9XBVNUVERGqdW/cQXYx33nmHm2++meHDh3PNNdcQGRnJ7NmzHfctFgvz5s3DYrEQFxfHPffcw3333cfLL7/swlq7RtsIf8wmyMgv4URuMUezCjmRW4zVbOKDe2Np1diP9NxiXlv4i6urKiIiUqvcuoeoMsuXL3d67+3tzfTp05k+ffo5P9OiRQu+//77Gq6Z+/P2sBDTyI/9J/JJSs0lp7AUgE5RgQT5evDXoZ0Z+X/rWfZL+gW+SUREpH6p8z1Ecmk6nB42++V4Domnh8t6RAcD0L1ZEACpOUUUlDSco01EREQUiBqYTk3OLL2vmFDds3kwAMG+ngSfnjt06KT2JRIRkYZDgaiB6RBZvvR+29FsdhwrX23WM/rMcSgxjfwAOHQqv/YrJyIi4iIKRA1MxZDZgRP5lJTZCfH1oEWYr+N+TFh5IDp4UoFIREQaDgWiBiYqyJsA7zNz6Xs2D8FkMjneV/QQKRCJiEhDokDUwJhMJscGjQA9T0+ortBSgUhERBogBaIGqOPpIzygvIfo1xxziBSIRESkAVEgaoAq5hGZTNAtOsjpXkUP0an8ErJP71MkIiJS3ykQNUC9W4ZgMZvo1TyEQG/nIzr8vaw0Dig/0kS9RCIi0lDUuZ2q5fK1CQ/g+8evJszfs9L7MY38OJFbzKFT+XT/zRwjERGR+kg9RA1U+8gAGvlXfrhtxdL7AyfUQyQiIg2DApGcJaaxNmcUEZGGRYFIztIyTCvNRESkYVEgkrNULL0/cDIfwzBcXBsREZGap0AkZ2kR5ovJBLlFZWTkl7i6OiIiIjVOgUjO4u1hISrIBzizY3VlPUWGYbBq70m2JGdSZrPXah1FRESqk5bdS6ViGvlxNKuQgyfzySooZeI3OxjctQkTb+7kKDM38ShPfrkVgABvK31iwhh9Xeuzdr8WERFxd+ohkkq1bOQLwL9WHeShf2/keHYR/1p1kE2HMwEoKbPz1qI9AHhazOQWlfFjUhovfrfLZXUWERGpKgUiqVRMI38AfknNxTAgKsgbgFfn78IwDL7YkMyRzELCA7zY/MIN/OeBPgDsPJpNUanNZfUWERGpCgUiqVTb8PJAZDLB80M6Mnd0P3w9LWxJzuKrjUeYumQfAI9d3xZ/Lyv92oTROMCLMrvB9qPZrqy6iIjIJVMgkkr1a9OICTd14LMHr+LBq1sRHujNwwNaA/Ds7G2czCsmOtSHO2OjATCZTPQ8fczHluRMV1VbRESkShSIpFIWs4k/D2hNXOswx7WHrm5FZKA39tMLzp66oR2e1jP/E7qiRflk6s2Hs2qzqiIiIpdNgUgumo+nhWcGtQegQ2QAt3Rv6nS/oodoc3KmNnQUEZE6Rcvu5ZLcfkUzIgO9aRPhj8VscrrXrVkwVrOJ9NxijmUX0TTYx0W1FBERuTTqIZJL1rdNI8IDvM+67uNpoWOTQAA2H9Y8IhERqTsUiKRa9WweDMCW5CyX1kNERORSKBBJtbri9C7Vm7XSTERE6hAFIqlWFYFo17Ecisu0QaOIiNQNCkRSraJDfQjz86TEZmfH0RxXV0dEROSiKBBJtTKZTI7DXbVBo4iI1BUKRFLtKiZWL9qZRpnN7trKiIiIXAQFIql2AztH4Gk1s/5QBs98vQ27XZs0ioiIe1MgkmrXJjyA90f0xGI2MXvzUV6et0s7V4uIiFtTIJIacWPnSN78QzcAPl59iJkrDri4RiIiIuemQCQ15raezXhxaCcA3lm8h/0n8lxcIxERkcopEEmNGtm3Jde2b0yJzc7zc3Zo6ExERNySApHUKJPJxMu3dMHLambNgVPMTTwKgN1ukJiSxZr9p9hwKINtR7Io1Yo0ERFxEZ12LzWueZgvj1/fljd+2M2r85I4mlnIlxtTSMkodCp3W8+mvHNnD9dUUkREGjT1EEmteOjqVrQJ9+dUfglvLtpDSkYhAV5W2oT7E9PID5MJ5mw5yqbDGa6uqoiINEAKRFIrPK1mXh/elWBfD3pEB/PG77ux/rl4fnxqAMuevpY7ekUD8PK8JO1bJCIitc5kaJbrBeXk5BAUFER2djaBgYGurk69lJ5bxHVvLCe/xMa7d/ZgWM+mrq6SiIjUcZfy+1s9ROIWwgO8efS6NgC8vvAXCktsLq6RiIg0JApE4jYe6B9D02AfjmcX8cr8Xdg0dCYiIrVEgUjchreHheeHdATgs3XJ/OnjDWQXlLq4ViIi0hAoEIlbualrE6aN6Im3h5kVe04w7O8/80tqjqurJSIi9ZwCkbidod2j+N/DfWka7MPBk/kMnbaKdxbvobhM84pERKRmKBCJW+rSNIhvxvQjvmM4pTaD95bs5eapq5i37ZgmXIuISLXTsvuLoGX3rmMYBvO3H+ev3+zkVH4JAH6eFm7sHMmEwR0ID/B2cQ1FRMRdadm91Bsmk4mbu0Xx41MDGH1da5qF+JBfYmPOlqOM/982V1dPRETqCbcORJMnT6Z3794EBAQQHh7OsGHD2L17t1OZoqIiRo8eTVhYGP7+/gwfPpy0tDSnMsnJyQwZMgRfX1/Cw8MZN24cZWVltdkUuUwhfp6MG9iBn565jn8/cCUAy/ec4EhmgYtrJiIi9YFbB6IVK1YwevRo1q5dy+LFiyktLeXGG28kPz/fUebJJ5/ku+++46uvvmLFihUcO3aM22+/3XHfZrMxZMgQSkpKWL16NZ988gkff/wxL7zwgiuaJJfJZDJxddvG9G0dhmHAfzekuLpKIiJSD9SpOUQnTpwgPDycFStWcM0115CdnU3jxo357LPP+P3vfw/AL7/8QseOHVmzZg1XXXUVCxYs4Oabb+bYsWNEREQAMHPmTMaPH8+JEyfw9PQ86znFxcUUFxc73ufk5BAdHa05RG5k3rZjjPlsCxGBXvw8/ndYLeXZ/vCpfJoE+eBpdeusLyIitaDeziHKzs4GIDQ0FIBNmzZRWlpKfHy8o0yHDh1o3rw5a9asAWDNmjV07drVEYYABg4cSE5ODjt37qz0OZMnTyYoKMjxio6OrqkmSRXd2CmSMD9P0nKKWfpLOoZh8Mq8XQx4Yzmxry7mqS8TWbwrjTqU90VExIXqTCCy2+2MHTuWfv360aVLFwBSU1Px9PQkODjYqWxERASpqamOMr8OQxX3K+5VZsKECWRnZzteKSkalnE3nlYzv+/VDIDP1ifz9uI9/GvVQQByisqYveUoD326kbcX73FlNUVEpI6oM4Fo9OjR7Nixgy+++KLGn+Xl5UVgYKDTS9zPXVc2B2D57hNMW7oPgBeHduK/f45jxJXlvXr/WnWQrIISx2d+Sc1h1Kcb2XE0u/YrLCIibqtOBKIxY8Ywb948li1bRrNmzRzXIyMjKSkpISsry6l8WloakZGRjjK/XXVW8b6ijNRNMY386Ns6zPF+wk0duL9fDFfGhDLptq50bBJIQYmNT9ccBqDUZueJzxNZtCuNCbO3azhNREQc3DoQGYbBmDFjmDNnDkuXLiUmJsbpfq9evfDw8GDJkiWOa7t37yY5OZm4uDgA4uLi2L59O+np6Y4yixcvJjAwkE6dOtVOQ6TGjPldG0J8PRg3sD1/HtDacd1kMvHwgFYAfLz6EIUlNj5ZfYjdabkAbD+azaJdaZV+p4iINDxuvcrs0Ucf5bPPPuObb76hffv2jutBQUH4+PgA8Mgjj/D999/z8ccfExgYyGOPPQbA6tWrgfJl9z169CAqKoopU6aQmprKvffey4MPPsikSZMuqh7aqbpuKrPZue6t5aRkFPLIta35dPUh8ktsdGoSyK7jOXSIDOD7x6/GbDa5uqoiIlID6s0qsxkzZpCdnc21115LkyZNHK8vv/zSUeadd97h5ptvZvjw4VxzzTVERkYye/Zsx32LxcK8efOwWCzExcVxzz33cN999/Hyyy+7oklSi6wWM6OuKe81mrF8P/klNq5oHsysB/sQ4GXll9Rcvt9x3MW1FBERd+DWPUTuQj1EdVdRqY3+ry/lZF4JZhN891h/OkcF8e6Pe3j3x720Cffn6Rvb8d2242w/ks2Lt3Tidx0iLvzFIiLi9upND5HI5fL2sPDItW0AePDqVnSOCgLgT/1jCPLxYF96Hg//ZzPztx0nOaOAv/x3Kydyi8/3lSIiUg8pEEm996d+LfnxqQFMuKmD41qgtwdj49sCEBXkzUNXx9AhMoDMglKen3tmBdr2I9m8+O1OliSlYbOrM1VEpL7SkNlF0JBZ/ZWeW0QjPy/MZhO7juVwy/urKLMbvHtnD1Jzinjzh92UnQ5CUUHe3N2nOQ8PaO04KkRERNzXpfz+ViC6CApEDcfUJXvP2t36ypah7EnPJaugFIDxgzrwyLWtK/u4iIi4Ec0hEqmiR65tTZem5f/R+HhYeH14V77881WsnXA9D5/e5+jHJO1fJCJS3ygQifyKh8XMh/fFMja+LfMf78+dvZtjMpnw9rCQ0Kf8qJDElCxyikqr/AzDMC7r8yIiUv0UiER+o0mQD2Pj29Gqsb/T9ehQX1qG+WKzG6zdf6rK3//q/CR6vLSIf689fLlVFRGRaqJAJHIJ+rdtBMCqfScd11buOcEfZq7mq40pFzwfbW9aLh/9fBC7ARPn7mDulqM1Wl8REbk4CkQil6B/m8bAmUBksxs8P3cHGw5lMu5/27j7w3UcOJF3zs+/vvAX7AaE+HoA8JevtvKjzlQTEXE5BSKRSxDXOgyzCQ6cyOdYViELd6SSnFGAn6cFbw8zaw6cYtB7P7Fsd/pZn1174BQ/JqVjMZv46uE4bu/ZFJvd4NHPNrNgu44QERFxJQUikUsQ5ONB9+hgAFbtPckHK/cD8MDVrVg0dgD92zSipMzO2C8SOZpV6Pic3W4w+fskAEZcGU2b8ACm/L4bN3aKoKTMziOzNvPagl+0+aOIiIsoEIlcoqvblM8jmrFiP1uPZONlNTMyrgXNw3z51/2xdG8WRHZhKWM+20ypzY5hGHy0+hBbj2Tj62nhievbAeWHz/494QpGXdMKgJkr9jPy/9ZrBZqIiAsoEIlcov5ty+cRHTyZD8DvezUjzN8LAC+rhffvvoJAbytbkrMY99VWbvv7al6ZtwuAhwe0pnGAl+O7rBYz/29wR6aN6ImPh4VV+04y6tONFJfZarlVIiINmwKRyCXqER2Mr6cFAJMJHrq6ldP96FBf3vhDdwDmJh4jMSULX08Lf7mhHY+eY4frod2j+O+f4/D3srL2QAZP/Xcr9osYPtMQm4hI9VAgErlEnlYzV7UKA2BQ50haNvI7q8zAzpE8/rs2eFhMjLgymuXjruWx69ue9wy0rs2CmHlPLzwsJuZvO86r85POuYy/1Gbn8c+3EPvqYjYeyqiehomINGA6y+wi6Cwz+a0dR7P5508HeHpge5qF+J6znM1uYDGbLum7v0k8yhNfJALw6rAu3HNVi7O+84kvtjBvW/nKtKbBPiwcezUB3h6X1ggRkXpOZ5mJ1LAuTYN4966e5w1DwCWHIYBbezRl3MD2ALz83S62H8l23LPbDcZ/vY15247jYTERHuDF0axCXvx21wW/91ReMav3n+REbvEl10lEpL6zuroCInK2R69tzZbkLH5MSuPRzzYxb8zVZBWW8Lf5SSzalYbZBFPv6knjAC/u+Mcavt58hOs7hjO4a5NKv2/etmP8v9nbySkqAyAqyJsrY0IZ87s2tAkPqM2miYi4JQ2ZXQQNmYkrZBeUcvP7P5GSUUjrxn4cPlVAmd3AbIK37ujObT2bAfDGD78wfdl+gnw8eHVYFwZ3beLomcouLOXVebv4atMRAEL9PMksKKHiv3qL2URCn+Y89ru2NPL3xGQykV1QysbDGWw4lInZBA/0j3GsogM4kllAXnEZHSL134KIuLdL+f2tQHQRFIjEVbYfyWb4jNWU2OwADGjXmGdv6kDHJmf+d1hSZuf3M1ez7fTQWsswX27oFMGW5CwSU7IosxuYTDD62jY8Ed+WolIb249m89HPh1j8m2NDPCwmSm3OPxJC/Tz569BOxLUOY+qSvXy+PgW7YfDR/b25tn14Df8NiIhUnQJRNVMgElf6YWcqc7cc5Z6rWtDv9KaQv5VTVMr/rTrIx6sPkVXgvLFjm3B/Xh3WxbEy7tfW7D/FpO+T2H402+l6q0Z+xLYMYduRbH5JzQXAajZR9qtl/o38PVnwxDVO+yqJiLgTBaJqpkAkdUVBSRlfrE9hd2ouPZsH069NI6JDzz/x2zAMcovLKCmzU2qz4221EOLnCZT3Ps1Yvp/3l+2l1GbQPTqYv9zQjknfJ/FLai7XtGvMx/f3ZvvRbP42P6k8mN3fm6hgH8f3p+cWkV9sI6aS7QlERGqSAlE1UyCShu7gyfLDbPu2DsNkMrE3LZeh76+iqNRO75YhbDyc6ZiX1KlJIF89HIefl5UNhzL440cbKCgp4+8JVzCoS+WTvn8tp6gUm81whDJ3UlBShofFjMd59pMSEfehQFTNFIhEzjZr3WGem7PD8f6W7lGs3n+Sk3kl3NgpgvviWvLQpxspLC0/hsTDYuJfI3tzTbvGGIbBtiPZFJbaiG0RgtVixmY3+PeaQ0z5YTdWs4k5o/vRurE/UN6LNW3pPvafyKNZiA/RIb54e1jILiwlp7CUHs2Dufr0kSo1JflUATdP+4kWYX58Meoq/Ly0SFfE3SkQVTMFIpGzGYbBq/OTSDqew9j4dlwZE8qmwxmM+HAdJWV2R7lr2jXG38vC99tT8fYw8+i1bfh++3HH3KTGAV4M7RbF1iNZbDqc6fhcuwh/5o7uh6+nldcX/sKM5fvPW5+/Du3EH/vFXFTd03OK2Hg4ky3JmXRpGsStPZpe8DNjv9jC3MRjAAzp1oT3R/TEZLr0faZEpPYoEFUzBSKRizd3y1HGfpkIQHzHCKYn9MSEiVH/3sjy3Scc5bw9zPh4WMj81SRwfy8rj1/fhg9/OsiJ3GJu69mUK5oHM/GbnQDc37clNrvBkcwCSmx2gnw8KCq1s/SXdABeuLkTf+p/7lB08GQ+Y7/YwtZfbXZpMsHXj/TliuYh5/zcrmM5DJn2E4ZxZnL5+EEdeOQcZ9OJiHtQIKpmCkQil+abxKMcySxk1DWtHPNtCktsPP7FFlIyCvhDbDS/v6IZvl4WVuw+wbdbj+HjYeGJ+LZEBfuw7sAp7v7nOqfDa5+Mb8cT8W3PepZhGLy5aDfTl5X3ID0zqD0PX9Ma8292CV++O53HP99CTlEZJhN0jAzEajGx7Ug2bcP9mfd4f7ysFmx2g8/XJ9M4wIsbO0VgMpn408cbWPpLOjd3a8JVrcJ4fu4OTCb4+I9XMqBdzQ7ViUjVKRBVMwUikdo3c8V+XlvwCwAjroxm0m1dzzlEZRgGby3aw/vL9gHQvVkQL93ahW5Ng0hKzWHB9lT+vnwfdgN6tQjh7wlXEBHoTWZ+CTe8s4KTeSU8fn1bRl3Tiic+38KS0z1Og7tGcmuPpvz535uwmE38+NQAWob5MmH2dr7YkEKgt5XvHutPi7AzK+hKbXYMo/wQ4N/KzC/h0zWH+WJDMp2aBPL/hnR0zJM6l/0n8nhn8R6ubR/O73s1q9LfpUhDpUBUzRSIRGqfYRi8t2QvJWV2nrqhHdYLrOwyDIN/rz3MGwt3k1tcfkRJgLeV3NPHlQDcGRvNy8M642W1OK7N33ac0Z9txmo2EdPIj73peXhazdjthtO+S3f3ac6k27oCUFxm464P1rIlOYv2EQHMfrQvfl5Wvt16jOdmb6ew1EbLRn60i/AnyMcTswkKS20s3JFKQYnN8Z0eFhN/6hfDmN+1qfRw3vnbjvPM/7aSf/ozj1/flifj27rl3KW0nCJmrUtmxJXRNAnyufAHRGqBAlE1UyASqTvSc4t4fcFuvt5cflyJn6eF3jGh3NojimE9mp4VJgzD4M//3sSi07t2N/L34sP7emE1m3nqv4nsTc/Dy2pmxbjriAzydnwuLaeIm6et4kRuMYO7RtIizO+CE78BOjYJZGRcC37Ymcqy03OqQv08GX1dGxL6NMfLamZfeh7/XnuYT9ccBso3yjxwMh+AEVc259VhXc55cPC+9Dwmf59ExyaB3BvXgojA8jrb7AbHsgppEuR9wXB5qfKLyxg+YzW/pOZyZUwoX466yi1DmzQ8CkTVTIFIpO7ZfyKP/OIyOjUJvGAASMsp4o5/rCHY15Ppd/ekWUj5ZpZFpTb+uzGFNo396VvJLuEbD2Uw4sO1TsedPDygNffGtWBvWi770vMoKLFhP/1j9ormIVzdtpEjLCz9JY1X5yU5wk6T04HreHaR0/c9fWM7vtiQwgvf7MBuQJ+YUP52W5ezDubdl57HXR+s5WReMVA+AXxg50jyisvYdDiTvOIyWjX24/khHbmuffh5Q0thiY11B09RWGLDfnoIsH+bRvh4WpzK2e0GD//nTKAEeO+uHhe1ck+kpikQVTMFIpH6z243zpqIfTH+s/Ywz8/dgZfVzJTfd7vkIFBms/O/TUd498e9pOaUByEvq5krY0L5U78Yrutw5ry4hTuOM/bLRIpK7XhYTDx0dStGXNmcqGAfDp7MZ8SHazmRW0z7iAACfaxsOJR5rsdyddtG/KlfDFe1CnOEnJIyO9uPZvO/TUeYt/WYY+ixQofIAP45MtYRGAHeWrSbaUv34WkxM7BLJN9tPUZEoBdL/nIt/qf3aiossbFsdzrztx9nT2oug7s24eEBrfHxtFBqs/PVxiMs/SWd++JacE0NTFIvtdk5kll40bul//OnA3y39Rjv3NmDVheY4yXuTYGomikQici5GIbB2gMZRAV7O02uvlRFpTZ+TEoj0NuDK2NC8fawVFouJaOAF7/d6Zj4DeW9N1aziYISGx0iA/jsoasI9fNk25Es5m8/TmSgN1fGhNIs2Je/r9jHR6sOOQ4M9rSa6RkdTFZBKQdO5jn1djUN9iEq2BuzycTe9Dwy8kto5O/JP+7tRVGpnf9uTOGb03szvfWH7gzp1oSB767k8KkC/jygFQlXtuDDnw7wv01HHBt0VmgS5M2dvaOZvfkoyRkFjusjrmzOc0M64u9lxW43KC6zn9UrdSlSMgoY9e9NJB3P4ZbuUbwyrAtBPmfP16qw/mAGd36wBsMoD4BzR/c757+FuD8FomqmQCQi7sQwDBbvSuPdH/eyNz3XEWJ+HYbO5/CpfD5YeYDlu09wNKvQ6V6Al5X4ThH8IbYZV8WEOXrNjmUV8uAnG9l1POes73v02tY8M6gDUD4M+KePN2IxmzAMg4p56c1CfBjStQkxjfx4f9k+jmSeeW4jfy/6tApl/rbjp9974mW1kJZTRJndoE9MKMOvaEa/to1Yf/AUi3elsSctj57RwVzXIZzYliHkFJaRml1EXnEprRr706qRHxsPZ/LorM1k5Jc4nhUV5M0LQztjsxvsS8+jzG7n3qtaEB7oTV5xGTe9t5KUjDN1u+eq5rw6rOvF/tM4OZJZQEZ+Cd2aBVfp8xVO5RWTW1RGS50HeMkUiKqZApGIuKuKydLpucV0bRpU6XL/czEMg/0n8tl8OJPGAV60iwwgKsj7nHOL8ovLeOKLRH5MSiPA28ot3aP4Q2w0PaKDnco98PEGRw/WgHaN+fM1rYg7fQ4elPeG/WvVQeZvO87Q7lGM7NsCX08ra/afYtz/tjqFparytJYfB2OzG3RpGsjjv2vLpO+TOHSq4Kyywb4evDqsCz/tOcmXG1NoFuLD/xvckUdnbQbg7wlXMLhrEwzDwDBwGlo1DIO84jJyi8rPufO0mNlxLJuPVx/ix6Q0DAPGDWzP6OvaAJBXXMYz/9vKjqM53HNVc+7u08IxtFhqs2M3jLNWQY7/ehsFJWW8dns37ugdfdl/NwA7jmZTXGajV4vQavk+d6VAVM0UiEREytntBruO59Am3P+cQ0lZBSX8d2MK/ds0plPUpf3MLCgpY+OhTPy9rUQGemOzG3y79Rhfbz7CgRP5tI8I4IZOEXRpGsSGQxks253OgRP5BHhZiQzyxtfTwt7Tk9kBbu0RxevDu+HtYSG/uIxJ3yexeFcaUcE+tA33Z9fxHHYeO9PrZTLB5w9dxVWtwhxHxnhazPh4WsgrLsNmNwjwshLo44HFbOJEbvFZw4GVGRvfljtio/nTxxscx9YABPl40L9NIw6czGd/eh6Y4Hftw7mlRxRrD5xyrDSs8PyQjjx4dSuna4ZhcOhUAdmFpYT6ehLi54G/l/WcwXbpL2k89OkmbHaDe65qzvNDOuHtYcEwDLYfzeZUXgmN/L1oFOBJblEZScdz2JOWS3pOMQUlNvJLymjT2J+HrmnlWMV4PkWlNqxmU7WvbrwYCkTVTIFIRMS1DMOgoMRW6aG6xWU2p14Vu90gOaOA4jI77SL8z7uartRmZ9rSfUxftg+b3eChq2N4bkgnx72ED9ex/lDGBetXcaQLlG/1MLxXM+6La8miXalMWbjbcT2/xEYjfy8eujqGLzemcOBE/gW/+5FrW1Nms/PhTwcBuL1nU0L8PCkosXEks4CtKVnkFJVV+lmTCdo09ueFoZ24um1jNidncveHaykqPXPeYLsIfwZ1acJ3W49x8OSF61PBy2rmvrgWDOvZFG8PC54WM3nFZaTlFJGeU8zOY9lsPJxJ0vEcArw9uKlLJEO7R9EnJrTWwpECUTVTIBIRqd92Hstmx9Fsbr+imeO4GSjv3dibloePpwV/LytWi4mcwlKyC0spsxs09vciPNALX08rhlG+mafZZHLaJ+qfPx3g1flJALSPCOBf95ev1LPZDZYkpbH/RD5twv3pEBlAblEZ3249xndbj1FcZuON33fnug7hGIbB35fv540fdldaf0+rmUZ+nmQWlJ6zx2pI1yb8vP8kWQWlDGjXmPv7tmTc/7Y5tmkA8PGw0LKRHxn5xZzKK8Hbw0L7yADaRwbQNNgHfy8rHhYzX28+4nQY86Wwmk1Eh/rSMsyXFmF+tAzzpWUjP1qG+dEizLda97BSIKpmCkQiInI55m07xvYj2Yz+XRsCK9mV/Lcqm68E8MPOVFbvO4m3pwVfDyuNA7zo1iyI9pEBTucGFpSUYQDFZXb++dMBPll9yDHBvXt0MJ8/1AdfTysn84p58dud5BaVcUv3KAZ1iXT0wlXEg8oCimEYLN9zgulL93HoVAElZTZKbHZ8Pa2EB3gRHuhNq0Z+xLYMoWfzEA6fzOe7bcdZuOO404HOv+braWHnSwMViNyZApGIiNRl245k8cq8XQD8497YC65ErCl2u0FabhEHT+Zz+FQBh07mc+hU+Z99PS3MfrRftT5PgaiaKRCJiIjUPZfy+7v2p3yLiIiIuBkFIhEREWnwFIhERESkwVMgEhERkQZPgUhEREQaPAUiERERafAUiERERKTBUyASERGRBq9BBaLp06fTsmVLvL296dOnD+vXr3d1lURERMQNNJhA9OWXX/LUU0/x17/+lc2bN9O9e3cGDhxIenq6q6smIiIiLtZgju7o06cPvXv35v333wfAbrcTHR3NY489xrPPPutUtri4mOLiM6f/5uTkEB0draM7RERE6hAd3fEbJSUlbNq0ifj4eMc1s9lMfHw8a9asOav85MmTCQoKcryio6Nrs7oiIiJSyxpEIDp58iQ2m42IiAin6xEREaSmpp5VfsKECWRnZzteKSkptVVVERERcQGrqyvgjry8vPDy8nJ1NURERKSWNIhA1KhRIywWC2lpaU7X09LSiIyMvODnK6ZZ5eTk1Ej9REREpPpV/N6+mOnSDSIQeXp60qtXL5YsWcKwYcOA8knVS5YsYcyYMRf8fG5uLoDmEomIiNRBubm5BAUFnbdMgwhEAE899RQjR44kNjaWK6+8knfffZf8/Hz++Mc/XvCzUVFRpKSkEBAQgMlkqoXa1q6KVXQpKSkNYhVdQ2pvQ2orqL31WUNqK6i91cUwDHJzc4mKirpg2QYTiO68805OnDjBCy+8QGpqKj169GDhwoVnTbSujNlsplmzZrVQS9cKDAxsEP/hVWhI7W1IbQW1tz5rSG0Ftbc6XKhnqEKDCUQAY8aMuaghMhEREWlYGsSyexEREZHzUSASvLy8+Otf/9pgthpoSO1tSG0Ftbc+a0htBbXXFRrM0R0iIiIi56IeIhEREWnwFIhERESkwVMgEhERkQZPgUhEREQaPAWiBmLy5Mn07t2bgIAAwsPDGTZsGLt373YqU1RUxOjRowkLC8Pf35/hw4efdf5bXfTaa69hMpkYO3as41p9a+vRo0e55557CAsLw8fHh65du7Jx40bHfcMweOGFF2jSpAk+Pj7Ex8ezd+9eF9a46mw2GxMnTiQmJgYfHx9at27NK6+84nRWUV1u78qVKxk6dChRUVGYTCbmzp3rdP9i2paRkUFCQgKBgYEEBwfzwAMPkJeXV4utuHjna29paSnjx4+na9eu+Pn5ERUVxX333cexY8ecvqO+tPe3Hn74YUwmE++++67T9brS3otpa1JSErfccgtBQUH4+fnRu3dvkpOTHfdr82e1AlEDsWLFCkaPHs3atWtZvHgxpaWl3HjjjeTn5zvKPPnkk3z33Xd89dVXrFixgmPHjnH77be7sNaXb8OGDfzjH/+gW7duTtfrU1szMzPp168fHh4eLFiwgF27dvHWW28REhLiKDNlyhSmTp3KzJkzWbduHX5+fgwcOJCioiIX1rxqXn/9dWbMmMH7779PUlISr7/+OlOmTGHatGmOMnW5vfn5+XTv3p3p06dXev9i2paQkMDOnTtZvHgx8+bNY+XKlYwaNaq2mnBJztfegoICNm/ezMSJE9m8eTOzZ89m9+7d3HLLLU7l6kt7f23OnDmsXbu20iMn6kp7L9TW/fv3079/fzp06MDy5cvZtm0bEydOxNvb21GmVn9WG9IgpaenG4CxYsUKwzAMIysry/Dw8DC++uorR5mkpCQDMNasWeOqal6W3Nxco23btsbixYuNAQMGGE888YRhGPWvrePHjzf69+9/zvt2u92IjIw03njjDce1rKwsw8vLy/j8889ro4rVasiQIcaf/vQnp2u33367kZCQYBhG/WovYMyZM8fx/mLatmvXLgMwNmzY4CizYMECw2QyGUePHq21ulfFb9tbmfXr1xuAcfjwYcMw6md7jxw5YjRt2tTYsWOH0aJFC+Odd95x3Kur7a2srXfeeadxzz33nPMztf2zWj1EDVR2djYAoaGhAGzatInS0lLi4+MdZTp06EDz5s1Zs2aNS+p4uUaPHs2QIUOc2gT1r63ffvstsbGx/OEPfyA8PJyePXvy4YcfOu4fPHiQ1NRUp/YGBQXRp0+fOtnevn37smTJEvbs2QPA1q1bWbVqFTfddBNQ/9r7axfTtjVr1hAcHExsbKyjTHx8PGazmXXr1tV6natbdnY2JpOJ4OBgoP611263c++99zJu3Dg6d+581v360l673c78+fNp164dAwcOJDw8nD59+jgNq9X2z2oFogbIbrczduxY+vXrR5cuXQBITU3F09PT8UOmQkREBKmpqS6o5eX54osv2Lx5M5MnTz7rXn1r64EDB5gxYwZt27blhx9+4JFHHuHxxx/nk08+AXC06bcHGdfV9j777LPcdddddOjQAQ8PD3r27MnYsWNJSEgA6l97f+1i2paamkp4eLjTfavVSmhoaJ1vf1FREePHj2fEiBGOA0DrW3tff/11rFYrjz/+eKX360t709PTycvL47XXXmPQoEEsWrSI2267jdtvv50VK1YAtf+zukEd7irlRo8ezY4dO1i1apWrq1IjUlJSeOKJJ1i8eLHTWHR9ZbfbiY2NZdKkSQD07NmTHTt2MHPmTEaOHOni2lW///73v8yaNYvPPvuMzp07k5iYyNixY4mKiqqX7ZVypaWl3HHHHRiGwYwZM1xdnRqxadMm3nvvPTZv3ozJZHJ1dWqU3W4H4NZbb+XJJ58EoEePHqxevZqZM2cyYMCAWq+TeogamDFjxjBv3jyWLVtGs2bNHNcjIyMpKSkhKyvLqXxaWhqRkZG1XMvLs2nTJtLT07niiiuwWq1YrVZWrFjB1KlTsVqtRERE1Ju2AjRp0oROnTo5XevYsaNjpUZFm367MqOutnfcuHGOXqKuXbty77338uSTTzp6A+tbe3/tYtoWGRlJenq60/2ysjIyMjLqbPsrwtDhw4dZvHixo3cI6ld7f/rpJ9LT02nevLnjZ9fhw4f5y1/+QsuWLYH6095GjRphtVov+LOrNn9WKxA1EIZhMGbMGObMmcPSpUuJiYlxut+rVy88PDxYsmSJ49ru3btJTk4mLi6utqt7Wa6//nq2b99OYmKi4xUbG0tCQoLjz/WlrQD9+vU7awuFPXv20KJFCwBiYmKIjIx0am9OTg7r1q2rk+0tKCjAbHb+0WWxWBz/j7O+tffXLqZtcXFxZGVlsWnTJkeZpUuXYrfb6dOnT63X+XJVhKG9e/fy448/EhYW5nS/PrX33nvvZdu2bU4/u6Kiohg3bhw//PADUH/a6+npSe/evc/7s6vWfy9V+zRtcUuPPPKIERQUZCxfvtw4fvy441VQUOAo8/DDDxvNmzc3li5damzcuNGIi4sz4uLiXFjr6vPrVWaGUb/aun79esNqtRp/+9vfjL179xqzZs0yfH19jf/85z+OMq+99poRHBxsfPPNN8a2bduMW2+91YiJiTEKCwtdWPOqGTlypNG0aVNj3rx5xsGDB43Zs2cbjRo1Mp555hlHmbrc3tzcXGPLli3Gli1bDMB4++23jS1btjhWVV1M2wYNGmT07NnTWLdunbFq1Sqjbdu2xogRI1zVpPM6X3tLSkqMW265xWjWrJmRmJjo9LOruLjY8R31pb2V+e0qM8OoO+29UFtnz55teHh4GB988IGxd+9eY9q0aYbFYjF++uknx3fU5s9qBaIGAqj09dFHHznKFBYWGo8++qgREhJi+Pr6Grfddptx/Phx11W6Gv02ENW3tn733XdGly5dDC8vL6NDhw7GBx984HTfbrcbEydONCIiIgwvLy/j+uuvN3bv3u2i2l6enJwc44knnjCaN29ueHt7G61atTKee+45p1+Qdbm9y5Ytq/S/1ZEjRxqGcXFtO3XqlDFixAjD39/fCAwMNP74xz8aubm5LmjNhZ2vvQcPHjznz65ly5Y5vqO+tLcylQWiutLei2nrv/71L6NNmzaGt7e30b17d2Pu3LlO31GbP6tNhvGr7V1FREREGiDNIRIREZEGT4FIREREGjwFIhEREWnwFIhERESkwVMgEhERkQZPgUhEREQaPAUiERERafAUiERERKTBUyASEbfVsmVL3n333Ysuv3z5ckwm01mHQYqIXIh2qhaRanPttdfSo0ePSwox53PixAn8/Pzw9fW9qPIlJSVkZGQQERGByWSqljpcquXLl3PdddeRmZlJcHCwS+ogIpfO6uoKiEjDYhgGNpsNq/XCP34aN258Sd/t6elJZGRkVasmIg2YhsxEpFrcf//9rFixgvfeew+TyYTJZOLQoUOOYawFCxbQq1cvvLy8WLVqFfv37+fWW28lIiICf39/evfuzY8//uj0nb8dMjOZTPzzn//ktttuw9fXl7Zt2/Ltt9867v92yOzjjz8mODiYH374gY4dO+Lv78+gQYM4fvy44zNlZWU8/vjjBAcHExYWxvjx4xk5ciTDhg07Z1sPHz7M0KFDCQkJwc/Pj86dO/P9999z6NAhrrvuOgBCQkIwmUzcf//9ANjtdiZPnkxMTAw+Pj50796d//3vf2fVff78+XTr1g1vb2+uuuoqduzYccHnisjlUyASkWrx3nvvERcXx0MPPcTx48c5fvw40dHRjvvPPvssr732GklJSXTr1o28vDwGDx7MkiVL2LJlC4MGDWLo0KEkJyef9zkvvfQSd9xxB9u2bWPw4MEkJCSQkZFxzvIFBQW8+eab/Pvf/2blypUkJyfz9NNPO+6//vrrzJo1i48++oiff/6ZnJwc5s6de946jB49muLiYlauXMn27dt5/fXX8ff3Jzo6mq+//hqA3bt3c/z4cd577z0AJk+ezKeffsrMmTPZuXMnTz75JPfccw8rVqxw+u5x48bx1ltvsWHDBho3bszQoUMpLS0973NFpBoYIiLVZMCAAcYTTzzhdG3ZsmUGYMydO/eCn+/cubMxbdo0x/sWLVoY77zzjuM9YDz//POO93l5eQZgLFiwwOlZmZmZhmEYxkcffWQAxr59+xyfmT59uhEREeF4HxERYbzxxhuO92VlZUbz5s2NW2+99Zz17Nq1q/Hiiy9Weu+3dTAMwygqKjJ8fX2N1atXO5V94IEHjBEjRjh97osvvnDcP3XqlOHj42N8+eWXF3yuiFwezSESkVoRGxvr9D4vL48XX3yR+fPnc/z4ccrKyigsLLxgD1G3bt0cf/bz8yMwMJD09PRzlvf19aV169aO902aNHGUz87OJi0tjSuvvNJx32Kx0KtXL+x2+zm/8/HHH+eRRx5h0aJFxMfHM3z4cKd6/da+ffsoKCjghhtucLpeUlJCz549na7FxcU5/hwaGkr79u1JSkqq0nNF5OJpyExEaoWfn5/T+6effpo5c+YwadIkfvrpJxITE+natSslJSXn/R4PDw+n9yaT6bzhpbLyxmUurn3wwQc5cOAA9957L9u3byc2NpZp06ads3xeXh4A8+fPJzEx0fHatWuX0zyi6n6uiFw8BSIRqTaenp7YbLaLKvvzzz9z//33c9ttt9G1a1ciIyM5dOhQzVbwN4KCgoiIiGDDhg2Oazabjc2bN1/ws9HR0Tz88MPMnj2bv/zlL3z44YdA+d9BxfdU6NSpE15eXiQnJ9OmTRun16/nWQGsXbvW8efMzEz27NlDx44dL/hcEbk8GjITkWrTsmVL1q1bx6FDh/D39yc0NPScZdu2bcvs2bMZOnQoJpOJiRMnnrenp6Y89thjTJ48mTZt2tChQwemTZtGZmbmefcxGjt2LDfddBPt2rUjMzOTZcuWOUJLixYtMJlMzJs3j8GDB+Pj40NAQABPP/00Tz75JHa7nf79+5Odnc3PP/9MYGAgI0eOdHz3yy+/TFhYGBERETz33HM0atTIseLtfM8VkcujHiIRqTZPP/00FouFTp060bhx4/POB3r77bcJCQmhb9++DB06lIEDB3LFFVfUYm3LjR8/nhEjRnDfffcRFxeHv78/AwcOxNvb+5yfsdlsjB49mo4dOzJo0CDatWvH3//+dwCaNm3KSy+9xLPPPktERARjxowB4JVXXmHixIlMnjzZ8bn58+cTExPj9N2vvfYaTzzxBL169SI1NZXvvvvOqdfpXM8VkcujnapFRH7FbrfTsWNH7rjjDl555ZVae652uBZxLQ2ZiUiDdvjwYRYtWsSAAQMoLi7m/fff5+DBg9x9992urpqI1CINmYlIg2Y2m/n444/p3bs3/fr1Y/v27fz444+amyPSwGjITERERBo89RCJiIhIg6dAJCIiIg2eApGIiIg0eApEIiIi0uApEImIiEiDp0AkIiIiDZ4CkYiIiDR4CkQiIiLS4P1/EDxy4MZqzhEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5449707291112293, recall = 0.24415832141154029, f1 = 0.33723036390581257\n",
      "precision = 0.49633967789165445, recall = 0.20759338640538885, f1 = 0.29274611398963735\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab41a9f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=32.73: 100%|█| 20/20 [01:16<00:00,  3.83s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABuAElEQVR4nO3dd5xU1f3/8dfMbC+zld1lYelll16VFewoTWzYEFui8acBC5YQv9HEkoiaxE40pmhMNJYIFhSVjvTee90FttC295n7+2Nm7u7ANpbtvJ+Pxz6yO/fOnXPQLG/P+ZxzLIZhGIiIiIi0UtamboCIiIhIQ1LYERERkVZNYUdERERaNYUdERERadUUdkRERKRVU9gRERGRVk1hR0RERFo1n6ZuQHPgdDo5evQooaGhWCyWpm6OiIiI1IJhGOTm5hIfH4/VWvX4jcIOcPToURISEpq6GSIiIlIHqamptG/fvsrrCjtAaGgo4PrDstvtTdwaERERqY2cnBwSEhLMv8erorAD5tSV3W5X2BEREWlhaipBUYGyiIiItGoKOyIiItKqKeyIiIhIq6aaHRERafUcDgelpaVN3Qw5S76+vthstnN+jsKOiIi0WoZhkJ6eTlZWVlM3ReooPDycuLi4c9oHT2FHRERaLU/QiYmJISgoSBvHtiCGYVBQUEBmZiYAbdu2rfOzFHZERKRVcjgcZtCJiopq6uZIHQQGBgKQmZlJTExMnae0VKAsIiKtkqdGJygoqIlbIufC88/vXGquFHZERKRV09RVy1Yf//wUdkRERKRVU9gRERGRVk1hR0REpJXr1KkTr7/+epM/o6loNVYjMQyDolIngX7nvjmSiIi0bpdddhkDBgyot3CxZs0agoOD6+VZLZFGdhrJLz9azwV/mMex3OKmboqIiLQChmFQVlZWq3vbtGlzXq9KU9hpBMVlDubtyCC3uIzdGblN3RwRkfOWYRgUlJQ1yZdhGLVq4z333MPixYt54403sFgsWCwWDh48yKJFi7BYLMyZM4fBgwfj7+/P0qVL2bdvH9dddx2xsbGEhIQwdOhQ5s2b5/XM06egLBYLf//737nhhhsICgqie/fufP3112f1Z5mSksJ1111HSEgIdrudW265hYyMDPP6pk2buPzyywkNDcVutzN48GDWrl0LwKFDhxg/fjwREREEBwfTu3dvvvvuu7P6/LOhaaxGsCs9l1KH61/y7EKdzSIi0lQKSx30+u0PTfLZ258fRZBfzX/tvvHGG+zevZs+ffrw/PPPA66RmYMHDwLw61//mj/96U906dKFiIgIUlNTGTt2LH/4wx/w9/fnww8/ZPz48ezatYsOHTpU+TnPPfccr7zyCn/84x956623mDRpEocOHSIyMrLGNjqdTjPoLF68mLKyMiZPnsytt97KokWLAJg0aRIDBw7knXfewWazsXHjRnx9fQGYPHkyJSUlLFmyhODgYLZv305ISEiNn1tXCjuNYNPhbPP7HIUdERGpRlhYGH5+fgQFBREXF3fG9eeff56rrrrK/DkyMpL+/fubP7/wwgvMmjWLr7/+milTplT5Offccw8TJ04E4MUXX+TNN99k9erVjB49usY2zp8/ny1btnDgwAESEhIA+PDDD+nduzdr1qxh6NChpKSk8OSTT5KYmAhA9+7dzfenpKQwYcIE+vbtC0CXLl1q/MxzobDTCLYczjK/18iOiEjTCfS1sf35UU322fVhyJAhXj/n5eXx7LPP8u2335KWlkZZWRmFhYWkpKRU+5x+/fqZ3wcHB2O3281zqGqyY8cOEhISzKAD0KtXL8LDw9mxYwdDhw7lscce47777uPf//43I0eO5Oabb6Zr164APPzwwzz44IP8+OOPjBw5kgkTJni1p76pZqcRbK44slOksCMi0lQsFgtBfj5N8lVfOzmfvqrqiSeeYNasWbz44ov89NNPbNy4kb59+1JSUlLtczxTShX/bJxOZ720EeDZZ59l27ZtjBs3jgULFtCrVy9mzZoFwH333cf+/fu588472bJlC0OGDOGtt96qt88+ncJOAysscXgVJWtkR0REauLn54fD4ajVvcuWLeOee+7hhhtuoG/fvsTFxZn1PQ0lKSmJ1NRUUlNTzde2b99OVlYWvXr1Ml/r0aMHU6dO5ccff+TGG2/k/fffN68lJCTwwAMPMHPmTB5//HH+9re/NVh7FXYa2Laj2TgrFODnFNZumaCIiJy/OnXqxKpVqzh48CDHjx+vdsSle/fuzJw5k40bN7Jp0yZuv/32eh2hqczIkSPp27cvkyZNYv369axevZq77rqLSy+9lCFDhlBYWMiUKVNYtGgRhw4dYtmyZaxZs4akpCQAHn30UX744QcOHDjA+vXrWbhwoXmtISjsNDDPFJZn9FIjOyIiUpMnnngCm81Gr169aNOmTbX1N6+++ioRERFcdNFFjB8/nlGjRjFo0KAGbZ/FYuGrr74iIiKCSy65hJEjR9KlSxc+/fRTAGw2GydOnOCuu+6iR48e3HLLLYwZM4bnnnsOAIfDweTJk0lKSmL06NH06NGDv/zlLw3XXqO2C/9bsZycHMLCwsjOzsZut9frsx/9ZANfbjxK33ZhbDmSzYCEcL6cPLxeP0NERM5UVFTEgQMH6Ny5MwEBAU3dHKmj6v451vbvb43sNLDNR1wjOyO6RwMqUBYREWlsCjsNKKeolP3H8gG4uJs77GgaS0REpFEp7DSgre5RnfYRgXSMdi0VzCms/ZbhIiIicu4UdhqQpzi5X/swwgJd+xmUOJwUlTZslbyIiJTTf2C2bPXxz69Jw84777xDv379sNvt2O12kpOTmTNnjnm9qKiIyZMnExUVRUhICBMmTPA6ZAxcW06PGzeOoKAgYmJiePLJJ2t9CmxD22KGnXCC/WzYrK4lWarbERFpeJ5N8woKCpq4JXIuPP/8Tt8E8Ww06XER7du356WXXqJ79+4YhsG//vUvrrvuOjZs2EDv3r2ZOnUq3377LZ9//jlhYWFMmTKFG2+8kWXLlgGupWvjxo0jLi6O5cuXk5aWxl133YWvry8vvvhiU3YNgE3uYyL6tQvDYrFgD/DhVEEp2YWlxNq1MkBEpCHZbDbCw8PNIxCCgoLqbRdjaXiGYVBQUEBmZibh4eHYbHU/bqPZLT2PjIzkj3/8IzfddBNt2rTh448/5qabbgJg586dJCUlsWLFCoYNG8acOXO45pprOHr0KLGxsQC8++67TJs2jWPHjuHn51erz2yIpeeGYfDRqhQ2H87i6Wt6YQ/w5dI/LuTQiQL+90AyQzrVfKqsiIicG8MwSE9PJysrq6mbInUUHh5OXFxcpUG1tn9/N5uDQB0OB59//jn5+fkkJyezbt06SktLGTlypHlPYmIiHTp0MMPOihUr6Nu3rxl0AEaNGsWDDz7Itm3bGDhwYKWfVVxcTHFxsflzTk5OvffHYrFwx7COQEfzNU/djjYWFBFpHBaLhbZt2xITE0NpqX73tjS+vr7nNKLj0eRhZ8uWLSQnJ1NUVERISAizZs2iV69ebNy4ET8/P8LDw73uj42NJT09HYD09HSvoOO57rlWlenTp5u7ODYme4Ar7KhmR0Skcdlstnr5S1NapiZfjdWzZ082btzIqlWrePDBB7n77rvZvn17g37mU089RXZ2tvlV8SCzhmSO7BQo7IiIiDSWJh/Z8fPzo1u3bgAMHjyYNWvW8MYbb3DrrbdSUlJCVlaW1+hORkYGcXFxAMTFxbF69Wqv53lWa3nuqYy/vz/+/v713JOa2QNdf9w5Rc1jtZiIiMj5oMlHdk7ndDopLi5m8ODB+Pr6Mn/+fPParl27SElJITk5GYDk5GS2bNliVtoDzJ07F7vd7nXEfHNhV82OiIhIo2vSkZ2nnnqKMWPG0KFDB3Jzc/n4449ZtGgRP/zwA2FhYdx777089thjREZGYrfbeeihh0hOTmbYsGEAXH311fTq1Ys777yTV155hfT0dJ5++mkmT57cJCM3NTFrdhR2REREGk2Thp3MzEzuuusu0tLSCAsLo1+/fvzwww9cddVVALz22mtYrVYmTJhAcXExo0aN8joC3mazMXv2bB588EGSk5MJDg7m7rvv5vnnn2+qLlVLq7FEREQaX7PbZ6cpNMQ+O5X5ZtNRHvrvBoZ1ieST+5Mb7HNERETOB7X9+7vZ1ey0ZuU1OypQFhERaSwKO43IM42lmh0REZHGo7DTiOwB7qXnCjsiIiKNRmGnEXlGdnKLy3A4z/tSKRERkUahsNOIPDU7ALk6MkJERKRRKOw0Il+blSA/19ksOSpSFhERaRQKO43Ms7Gg9toRERFpHAo7jcxckaVpLBERkUahsNPIPIeBamRHRESkcSjsNDLttSMiItK4FHYamWp2REREGpfCTiOzq2ZHRESkUSnsNDK7Tj4XERFpVAo7jay8Zse1z85b8/dw/4drKXU4m7JZIiIirZZPUzfgfOM5Hyu7sJTMnCJem7cbpwGbD2czuGNEE7dORESk9dHITiOruM/OlxuP4DkiK7uwpAlbJSIi0nop7DSyijU7X6w7Yr6eVaAaHhERkYagaaxG5hnZOXSiwOvkc4UdERGRhqGRnUbmGdmpGHQAsgo0jSUiItIQFHYamWdkx6N3vB2ALC1FFxERaRAKO40s2M+GzWoBIDrEj2v7xwOaxhIREWkoCjuNzGKxmMvPrxvQjqgQf0AjOyIiIg1FYacJJLW14+dj5dahCYR7VmepZkdERKRBaDVWE3j3zsFkF5SSEBlknn6ukR0REZGGobDTBOwBvubp5+FBrv9VzY6IiEjD0DRWEwsL9ANcOyqfvhxdREREzp3CThPzLEU3DMgt0uiOiIhIfVPYaWJ+PlZC/F2ziac0lSUiIlLvFHaaAc/ojnZRFhERqX8KO82AWaSsFVkiIiL1TmGnGfCEnWxNY4mIiNQ7hZ1mINy9IkvTWCIiIvVPYacZCNM0loiISINR2GkGwgO1saCIiEhDUdhpBiKCXNNY2RrZERERqXcKO82AZxrrlGp2RERE6p3CTjOgaSwREZGGo7DTDIRrGktERKTBKOw0A+Unn2saS0REpL4p7DQDnmms7MJSnDr5XEREpF4p7DQDngJlpwG5xWVN3BoREZHWRWGnGfD3sRHkZwN0ZISIiEh9U9hpJswVWYWq2xEREalPCjvNRFiQ53wsjeyIiIjUJ4WdZsIzsqONBUVEROqXwk4z4Vl+rr12RERE6pfCTjNRvteOwo6IiEh9UthpJsICVbMjIiLSEJo07EyfPp2hQ4cSGhpKTEwM119/Pbt27fK657LLLsNisXh9PfDAA173pKSkMG7cOIKCgoiJieHJJ5+krKxl7VcTEaTVWCIiIg3Bpyk/fPHixUyePJmhQ4dSVlbG//3f/3H11Vezfft2goODzft+8Ytf8Pzzz5s/BwUFmd87HA7GjRtHXFwcy5cvJy0tjbvuugtfX19efPHFRu3PuTBrdjSyIyIiUq+aNOx8//33Xj9/8MEHxMTEsG7dOi655BLz9aCgIOLi4ip9xo8//sj27duZN28esbGxDBgwgBdeeIFp06bx7LPP4ufn16B9qC/mNJYKlEVEROpVs6rZyc7OBiAyMtLr9Y8++ojo6Gj69OnDU089RUFBgXltxYoV9O3bl9jYWPO1UaNGkZOTw7Zt2yr9nOLiYnJycry+mpoOAxUREWkYTTqyU5HT6eTRRx9l+PDh9OnTx3z99ttvp2PHjsTHx7N582amTZvGrl27mDlzJgDp6eleQQcwf05PT6/0s6ZPn85zzz3XQD2pG63GEhERaRjNJuxMnjyZrVu3snTpUq/X77//fvP7vn370rZtW6688kr27dtH165d6/RZTz31FI899pj5c05ODgkJCXVreD0JrzCNZRgGFoulSdsjIiLSWjSLaawpU6Ywe/ZsFi5cSPv27au998ILLwRg7969AMTFxZGRkeF1j+fnqup8/P39sdvtXl9NzTOy43Aa5OnkcxERkXrTpGHHMAymTJnCrFmzWLBgAZ07d67xPRs3bgSgbdu2ACQnJ7NlyxYyMzPNe+bOnYvdbqdXr14N0u6GEOBrI8DX9Y9DU1kiIiL1p0mnsSZPnszHH3/MV199RWhoqFljExYWRmBgIPv27ePjjz9m7NixREVFsXnzZqZOncoll1xCv379ALj66qvp1asXd955J6+88grp6ek8/fTTTJ48GX9//6bs3lmLCPIjLbuIk/klJEQG1fwGERERqVGTjuy88847ZGdnc9lll9G2bVvz69NPPwXAz8+PefPmcfXVV5OYmMjjjz/OhAkT+Oabb8xn2Gw2Zs+ejc1mIzk5mTvuuIO77rrLa1+elqJNqCucHcstbuKWiIiItB5NOrJjGEa11xMSEli8eHGNz+nYsSPfffddfTWrycS4w06mwo6IiEi9aRYFyuLSJjQAgMzcoiZuiYiISOuhsNOMeEZ2MnI0siMiIlJfFHaakRi7p2ZHIzsiIiL1RWGnGYkxp7E0siMiIlJfFHaaEbNAWdNYIiIi9UZhpxnxTGMdzyvG6ax+pZqIiIjUjsJOMxId4o/FAmVOg5M6/VxERKReKOw0I742K1HBrgNBNZUlIiJSPxR2mhnttSMiIlK/FHaamZp2UX7um23c+Y9VlDmcjdksERGRFqtJj4uQM5WvyDpzZMcwDP6z8hClDoPdGXn0irc3dvNERERaHI3sNDOeFVmVjezkFZdR6nCt0jqep5oeERGR2lDYaWbMjQUrKVA+lV9qfq+wIyIiUjsKO81Mec3OmdNYFZejK+yIiIjUjsJOM1PdNNap/PKwcyJP+/CIiIjUhsJOM1PxfCzD8N5F+USFsHNMIzsiIiK1orDTzLRxT2OVlDnJKSzzulZxZOe4RnZERERqRWGnmQnwtWEPcO0IcHrdjlfNjk5GFxERqRWFnWYo1u6ayso4bUWW98iOwo6IiEhtKOw0Q+VFyqeN7FQsUM4v0cnoIiIitaCw0wxVLFKu6FSFaSyH0yCrsBQRERGpnsJOM1R+ZIR32Km4Ggs0lSUiIlIbCjvNUJsqNhb01OzYrBZARcoiIiK1obDTDMXYz5zGqjht1SkqCNBeOyIiIrWhsNMMeaaxjlUIO9mFpXj2GOweEwporx0REZHaUNhphsprdsqnsTwrsewBPsSFuUZ+VLMjIiJSM4WdZsgzjZVf4iCv2LWLsmclVlSIP9EhfoBqdkRERGpDYacZCvH3IcjPBkCGe3THc/BnRJAv0SGukR+N7IiIiNRMYaeZSohwFSGnnCwAykd2IoP9KoQd1eyIiIjURGGnmeocHQzAgWP5QHnNTkSQH9GhGtkRERGpLYWdZqpzG1fYOXjCFXY8e+y4RnZcNTsn8kowDB0ZISIiUh2FnWbKHNk57h7ZqWQaq8ThJKewrGkaKCIi0kIo7DRTnrCz/5j3yE5EsB8BvjZC/X0AbSwoIiJSE4WdZsoTdo5mF1JU6jBrdiKDXFNYqtsRERGpHYWdZioq2I/QAB8MAw6dKDCnsSKC3WHHs9eOwo6IiEi1FHaaKYvFQhezbiePU/muc7EizbDjHtnRxoIiIiLVUthpxjxTWbvS88ydlM1pLO21IyIiUisKO81Y5+gQANannALAZrVgD3QVJmsXZRERkdpR2GnGPHvteMJORJAfFosFgOhQT82ORnZERESqo7DTjHWOcoWd3CL3FFawr3lNIzsiIiK1o7DTjHWKDvL6OcJdrwMKOyIiIrWlsNOMhQb40sa9nw6Ur8QCaFMh7OjICBERkaop7DRznhVZ4B12PDU7RaVO8kscjd4uERGRlkJhp5nrUkXYCfLzIdDXBsAJTWWJiIhUSWGnmas4slOxZgcgLNBVsKzDQEVERKqmsNPMVTWNBeVhJ7uwtFHbJCIi0pIo7DRzXiM7p4UdzwaDOUUKOyIiIlVR2GnmOkQF4d5H0DwqwkMjOyIiIjVr0rAzffp0hg4dSmhoKDExMVx//fXs2rXL656ioiImT55MVFQUISEhTJgwgYyMDK97UlJSGDduHEFBQcTExPDkk09SVtY66lj8fWxc3SuWhMhAusWEeF2zB3hqdhR2REREqtKkYWfx4sVMnjyZlStXMnfuXEpLS7n66qvJz88375k6dSrffPMNn3/+OYsXL+bo0aPceOON5nWHw8G4ceMoKSlh+fLl/Otf/+KDDz7gt7/9bVN0qUH89c4hLH7icgL9bF6v2z0FyprGEhERqZLFaEY70h07doyYmBgWL17MJZdcQnZ2Nm3atOHjjz/mpptuAmDnzp0kJSWxYsUKhg0bxpw5c7jmmms4evQosbGxALz77rtMmzaNY8eO4efnV91HApCTk0NYWBjZ2dnY7fYG7WN9enXubt6cv4c7hnXg99f3bermiIiINKra/v3drGp2srOzAYiMjARg3bp1lJaWMnLkSPOexMREOnTowIoVKwBYsWIFffv2NYMOwKhRo8jJyWHbtm2Vfk5xcTE5OTleXy2RPcBdoKyl5yIiIlVqNmHH6XTy6KOPMnz4cPr06QNAeno6fn5+hIeHe90bGxtLenq6eU/FoOO57rlWmenTpxMWFmZ+JSQk1HNvGocKlEVERGrWbMLO5MmT2bp1K5988kmDf9ZTTz1Fdna2+ZWamtrgn9kQVLMjIiJSM5+mbgDAlClTmD17NkuWLKF9+/bm63FxcZSUlJCVleU1upORkUFcXJx5z+rVq72e51mt5bnndP7+/vj7+1d6rSXRyI6IiEjNmnRkxzAMpkyZwqxZs1iwYAGdO3f2uj548GB8fX2ZP3+++dquXbtISUkhOTkZgOTkZLZs2UJmZqZ5z9y5c7Hb7fTq1atxOtJEypeeq2ZHRESkKk06sjN58mQ+/vhjvvrqK0JDQ80am7CwMAIDAwkLC+Pee+/lscceIzIyErvdzkMPPURycjLDhg0D4Oqrr6ZXr17ceeedvPLKK6Snp/P0008zefLkVjF6U52wIE1jiYiI1KRJw84777wDwGWXXeb1+vvvv88999wDwGuvvYbVamXChAkUFxczatQo/vKXv5j32mw2Zs+ezYMPPkhycjLBwcHcfffdPP/8843VjSbjWY1VUuakqNRBgK+thneIiIicf5rVPjtNpaXus+N0GnT7zXc4DVj9f1cSYw+o03P+/OMuHE6DJ0f1xOI5m0JERKSZa5H77MjZsVot5oqsuhYpH8st5q0Fe/nLon0cOJ5f8xtERERaGIWdFs4sUq5j3c7BE+UBZ9m+E/XSJhERkeZEYaeFO9fl5weOlYed5XuP10ubREREmhOFnRbOHnhuR0YcqDCys2L/CZzO876ES0REWhmFnRauppGd//fvtdzy1xWUOZyVXj9YoU4nq6CU7Wkt85wwERGRqijstHDlGwueGXayCkr4YVsGqw+cZE9mXqXv9xQlh/i7RoiWaSpLRERaGYWdFi6smvOxDp4oML/fUcmIjWEYHHLfc8PAdoCKlEVEpPVR2Gnhqlt6XnGKamd67hnXM3KKKSx1YLNauHWo6+T3NQdOUlJW+ZSXiIhIS6Sw08KZJ59XUqBccVl5ZSM7nims9hGB9I63Ex3iR2Gpg42pWQ3TWBERkSagsNPCeY6MqGlkZ0famSM7njDUKSoYi8VCctdoQHU7IiLSuijstHD2WtbsHM8r5lhusfd1dxjqHB0MwPCuUQAs36ewIyIirYfCTgtX3dJzz8iNr8113tXOdO+pLM80VqeoIACGd3ON7GxIySKroKRhGiwiItLIFHZauKqWnmcVlJBV4HrtIvf01M7TprLMaSz3yE5CZBBJbe2UOQ2+2ZzWoO0WERFpLAo7LZxnZCe3uMxr92PPFFas3Z/BHSMA2FFhZMfpLF927pnGApgwyLUEfeb6ww3bcBERkUaisNPCeY6LMAxX4PE45B616RgVTGJcKOBdpJyWU0RxmRMfq4V24YHm69cOiMdmtbAhJYt9xyrfiFBERKQlUdhp4fx9bAT4uv4xVpzKOnjcPWoTFUxSWzsAezNzKXUfG+EpTu4QGYSPrfxfg5jQAC7p7pr2mrX+SMN3QEREpIEp7LQCnrqdikXKnnqcjtFBtI8IJMTfh1KHYY7WmMXJFaawPCYMbg/ArA1HdDCoiIi0eHUKO//617/49ttvzZ9/9atfER4ezkUXXcShQ4fqrXFSO5UdGeEJO53de+h4prI8RcoHj5fvsXO6kUmxhAb4cCSrkJUHdHyEiIi0bHUKOy+++CKBga46jxUrVjBjxgxeeeUVoqOjmTp1ar02UGpWvotyxWms8podwJzK8hQpm2EoOuiM5wX42rimXzwAX6zTVJaIiLRsdQo7qampdOvWDYAvv/ySCRMmcP/99zN9+nR++umnem2g1Myzi7LnyIjsglJOuZedd3TvoZPY1rtIubppLICbBrtWZc3ZmqazskREpEWrU9gJCQnhxAnX9MaPP/7IVVddBUBAQACFhYX11zqpldM3FvSM2sSE+hPs7wpCiXGukZ01B05y2R8Xsu9Y1dNYAIM6RBAR5EtBiYPtlZyrJSIi0lLUKexcddVV3Hfffdx3333s3r2bsWPHArBt2zY6depUn+2TWjj9yIiKZ155JMaF4mO1UFjq4OCJAiwWuLpXLO0jAs98IGCxWBjYwbU/z/pDpxqy+SIiIg3Kpy5vmjFjBk8//TSpqal88cUXREW5zlRat24dEydOrNcGSs3CTqvZ8Sw771ShHifY34e3Jg5kR1oOAztGMKhDhPm+qgzqEM6CnZmsTznFz+ncQK0XERFpWHUKO+Hh4bz99ttnvP7cc8+dc4Pk7J2+9LzihoIVjenbljF929b6uYPcIzsbUrK8Xs8uLMVmtRDiX6d/fURERBpVnaaxvv/+e5YuXWr+PGPGDAYMGMDtt9/OqVOa8mhs5UvPXQXKB8yVVpXX49RW/4RwrBY4klVIRk4R4Cp+vvLPi7n2raXmBoUiIiLNWZ3CzpNPPklOjqtodcuWLTz++OOMHTuWAwcO8Nhjj9VrA6VmniMjsgtLKXU4zZVWnpVYdRXs70NPd2Gzp27n2y1pHM8rZv/xfBbvOnZOzxcREWkMdQo7Bw4coFevXgB88cUXXHPNNbz44ovMmDGDOXPm1GsDpWYV99n5YNlBsgpKCQ/ypWubkHN+9qAO4QCsT3GFnS83lu+7M3ODDgsVEZHmr05hx8/Pj4ICVxHsvHnzuPrqqwGIjIw0R3yk8XhqdtKyi3ht3m4AnhqTSICv7Zyf7anbWZ+SxeFTBaw+cNK8Nm97JtkFpVW9VUREpFmoU9gZMWIEjz32GC+88AKrV69m3LhxAOzevZv27dvXawOlZp6anbziMgpKHAzuGMHNgxPq5dmDOrrCzpbD2Xy+1jWSc1HXKBLjQilxOJm95Wi9fI6IiEhDqVPYefvtt/Hx8eF///sf77zzDu3auXfbnTOH0aNH12sDpWb2CkvIbVYLv7++D1arpV6e3SkqiMhgP0ocTv72034Arh/YjgmDXKF2Zh1PRjcMg7cX7GHOlrR6aaeIiEhV6rR2uEOHDsyePfuM11977bVzbpCcvVB/HywWMAz4+fBO5jlY9cFisTAwIZz5OzMpKHHg72NldJ84ikocTJ+zg3WHTnHweH6Vx05UZXdGHn/6cTf+PlYu7dmGID8tYxcRkYZRp5EdAIfDwRdffMHvf/97fv/73zNr1iwcDkd9tk1qyWq1cNvQDiR3ieLRkT3q/fmeqSyAkb1isQf4EmMP4OLubQCYueHsR3cyc11L2YvLnCzZrVVdIiLScOoUdvbu3UtSUhJ33XUXM2fOZObMmdxxxx307t2bffv21XcbpRam39iX/94/zDwLqz4NdK/IArhhQDvz+xsHub6fteEwhmGc1TNPVShs/mFbxrk1UEREpBp1CjsPP/wwXbt2JTU1lfXr17N+/XpSUlLo3LkzDz/8cH23UZrYwIQI2oYF0CU6mEt6tDFfv6pXLDarhdSThaS7Nx2srayCEvP7+TsytEGhiIg0mDoNAyxevJiVK1cSGRlpvhYVFcVLL73E8OHD661x0jwE+tmY99ilAPj5lOfjID8fuseEsDM9l02p2bQNq/xQ0cqczC8POzlFZazaf5IR3aPrr9EiIiJudRrZ8ff3Jzc394zX8/Ly8PPzO+dGSfMT7O9T6RRZ//bhAGw+nHVWz8s6bX+eH7al17VpIiIi1apT2Lnmmmu4//77WbVqFYZhYBgGK1eu5IEHHuDaa6+t7zZKM9YvIQyAzYezz+p9npGdEd1cozk/bk/H6Ty7uh8REZHaqFPYefPNN+natSvJyckEBAQQEBDARRddRLdu3Xj99dfruYnSnFUc2TmbIuVT7pqdcf3aEuxnIyOnmM1Hzi4wiYiI1EadanbCw8P56quv2Lt3Lzt27AAgKSmJbt261WvjpPnrGReKn4+VnKIyDp0oqPV+O55prFi7P5clxvDt5jR+2JbOgITwBmytiIicj2oddmo6zXzhwoXm96+++mrdWyQtiq/NSq+2djamZrHpcFatw45nGisiyI9RveP4dnMa83dkMG10YkM2V0REzkO1DjsbNmyo1X0WS/0cUyAtR//2YWxMzWLz4Wyuq7APT3U8S88jgvyICwsAYP+xfMocTnxsdd7rUkRE5Ay1DjsVR25EKurXPhw4ZK7IMgyDT9akEhXsx9W94864v7jMQX6Ja7ftiCA/QgN88POxUlLmJC27iITIoMZrvIiItHr6T2g5Z/3dK7K2HsmhzOHkuy3pPDVzC/f/ex3fbj7zoE9PvY7NaiE0wAer1UJChGuPnpSTBY3XcBEROS8o7Mg56xwdQrCfjcJSB5sOZ/PC7O3mtamfbWTNwZNe93tWYoUH+pqns3dwj+Yo7IiISH1T2JFzZrNa6NPONbrz+GcbSc8pIiEykJFJMZSUOfnFh2vZdyzPvN9TnBwe5Gu+prAjIiINRWFH6kV/95LxgydcYeW5a3vz1sRB9E8IJ6uglEc+KS9w90xjRQaX77btqdNJOaGwIyIi9UthR+pFv/Zh5vcjk2K5IjGWQD8bM24fCLjqeYrLXEXJ5jRWUHnY6RjlWrKukR0REalvCjtSLwZ1iMBmteDvY+V343uZr7cLDyTQ1wbA0SzXyeinzD12NI0lIiINr0nDzpIlSxg/fjzx8fFYLBa+/PJLr+v33HMPFovF62v06NFe95w8eZJJkyZht9sJDw/n3nvvJS8vD2lc8eGB/PvnF/DFgxd5LR23WCy0d6+0OnzKFWROuaexIrymsVz3ZBeWkn3aIaEiIiLnoknDTn5+Pv3792fGjBlV3jN69GjS0tLMr//+979e1ydNmsS2bduYO3cus2fPZsmSJdx///0N3XSpxEXdos1C5YrKw04hUHFkpzzsBPn50CbUH9DojoiI1K86nY1VX8aMGcOYMWOqvcff35+4uDM3pgPYsWMH33//PWvWrGHIkCEAvPXWW4wdO5Y//elPxMfH13ub5ey1j3CN9JSP7LjCTmSFsAOuqaxjucWknCygb/szQ5OIiEhdNPuanUWLFhETE0PPnj158MEHOXHihHltxYoVhIeHm0EHYOTIkVitVlatWlXlM4uLi8nJyfH6kobjGdk54hnZcU9TVVx6DuV1O4dO5jdi60REpLVr1mFn9OjRfPjhh8yfP5+XX36ZxYsXM2bMGBwO16qe9PR0YmJivN7j4+NDZGQk6enpVT53+vTphIWFmV8JCQkN2o/zXfnIjifsuKexgr1Hdjy1PqnuaazMnCKGvTifxz/b1FhNFRGRVqhZh53bbruNa6+9lr59+3L99dcze/Zs1qxZw6JFi87puU899RTZ2dnmV2pqav00WCpVm5odgI6nrcj6etNR0nOK+HrTEXPZuoiIyNlq1mHndF26dCE6Opq9e/cCEBcXR2Zmptc9ZWVlnDx5sso6H3DVAdntdq8vaTiesJORW0RhiYOcojLAe+k5QIco9zSWe2PB77a4ztUqdRhsP6qpRhERqZsWFXYOHz7MiRMnaNu2LQDJyclkZWWxbt06854FCxbgdDq58MILm6qZcprIYD8CfW0YBmxPyzZfDwusvGbnaFYhqScLWJ+SZV7blJqFiIhIXTTpaqy8vDxzlAbgwIEDbNy4kcjISCIjI3nuueeYMGECcXFx7Nu3j1/96ld069aNUaNGAZCUlMTo0aP5xS9+wbvvvktpaSlTpkzhtttu00qsZsSz186ezDy2HHaFnbBAX3xs3lk7JtQffx8rxWVO/rH0gNe1jQo7IiJSR006srN27VoGDhzIwIGuIwUee+wxBg4cyG9/+1tsNhubN2/m2muvpUePHtx7770MHjyYn376CX9/f/MZH330EYmJiVx55ZWMHTuWESNG8N577zVVl6QKnqmszUdcYef0KSxwhSLP6M6na1x1VJf0aAPApsPZZ9wvIiJSG006snPZZZdhGEaV13/44YcanxEZGcnHH39cn82SBuBZkbXVHXbCTytO9ugQGcSezDwKS10FydNG92TJ7mMcOJ5PVkFJle8TERGpSouq2ZGWyzOyszfTdZRHZHDloaXiURMDEsLpHR9GJ3fhskZ3RESkLhR2pFF4Rnac7oG80zcU9OgYVR52xvRxrajrnxAOqEhZRETqRmFHGkU798iOx+lHRXh0iKwYdlyr7vq3DwdUpCwiInXTpDU7cv5of1rYOX33ZI8BCeGEBfoypGOEue/OgA7hgGtkxzAMLBZLg7ZVRERaF4UdaRRRwX4E+FopKnUCVU9jRYX4s/bpkV6v9Wprx9dm4UR+CYdPFXrV9YiIiNRE01jSKFx77ZSHlKqmsQB8bVZ8K+zBE+BrI6mta5drTWWJiMjZUtiRRlNxKutsl5B76nYaqkg5t6iURz/ZwIKdGQ3yfBERaToKO9JoKoadiODKp7Gq4lmR1VAjO99sSuPLjUd57LNN5BaVNshniIhI01DNjjSa2k5jVWaQu0h585Fsissc+PvYAMgvLuPRTzeSW1RKfFggbcMDuH5AO7rHhp7V83emuw4azSoo5YNlB3noyu5n9X4REWm+FHak0ZzLNFbn6GCiQ/w4nlfC5sPZDO0UCcD3W9OZu9176umnPcf5esoIr9eOZhUSEeRHoJ+t0ufvTM81v//bT/u5e3gn7AFnN/okIiLNk6axpNF4RnaC/Wz4+Zzdv3oWi8UMOKsPnDRfX7b3OACje8fxiHs0ZvvRHIrLHOY9245mc/ErC3nyf5sqfbZhGOxMc43shAf5klNUxvtLDwKwcv8Jfvb+auZtVy2PiEhLpbAjjaZPvJ2resVy38Vd6vT+Czp7hx3DMFjqDjt3XdSRR0d2xx7gQ5nTMI+lAFi8+xgOp8Hc7RmUlDnPeG5adhE5RWXYrBZ+N74XAH9fup+nZm7mtvdWsnDXsTNOYRcRkZZDYUcajY/Nyt/uGsLUq3rU6f2ekZ11h07hcBrsycwjM7eYAF8rgzpEYLFY6BXvWqK+I618Wspz+GhxmZPNh7POeK6nXqdrm2Cu69+OnrGh5BaV8d/VqeY9x/KK69RmERFpego70mIktbUT6u9DXnEZO9JyWLrHNaoztFMkAb428x5wTWV5bDlSfoDoqgpTYB6eep3EODtWq4VpY3pitbjO6Xrhut4AHMtV2BERaakUdqTFsFktDO4UAbimsjz1OiO6RZv3eMLOjjTP6qoSUk8WmtcrDTvuUaDEtq4VXFckxrLs11cwd+qljO8fD0B2YSlFpY4z3isiIs2fwo60KJ66neX7jrNy/wkAhlcIO708IztpORiGYY7q+LsLotcdPEmZw7tuxzONlRhXvly9bVggfj5WwgJ98XPv5nxcU1kiIi2Swo60KBe463YW7Mwkv8RBRJCvGXAAuseG4GO1kF1YSlp2kRl2rkyKwR7gQ36Jg20VpriKyxzsP5YPuKaxTmexWGgT6g9UPZVVUubkH0sPcCSrsNLrIiLStBR2pEXp2z4Mfx8rTsP180XdorFay09B9/ex0bVNCOCq2/EUJ/dvH17p0vV9mfmUOQ3sAT60DQuo9DOjawg7n65N5YXZ2/n1F5vPrXMiItIgFHakRfH3sTHAfXQEeNfreJSvyMph82FX2OnbLowLu7jCTsW6nfIpLDsWi4XKtAlxh50qprE2HDoFwPJ9JzihqS4RkWZHYUdanAvddTtQedhJchcaL993gsOnXFNLvduFcUHnKADWHDyJ0z00tCvduzi5MjVNY2096gpUDqfBD9u0+aCISHOjsCMtjqcguUubYBIig8643qttGAAr3AXMnaKCCAv0pU+8nSA/G9mFpezKcIWcHRWWnVelurBTUFLmtYHhd1vS6tIlERFpQAo70uJc2CWKGbcP4t07Bld6Pem0UZo+7Vzhx8dmZXBH19L1Ve4g5DkmomdczSM7mZWEnR1puTgNCHKfubViv6ayRESaG4UdaZHG9WtLjypONo8K8SfGHVDAVa/jMayLayrrvSX7+fOPu8wAU23YCal6ZMdTAD2sSxR92tk1lSUi0gwp7Eir5ClSBtcKLo+re8US6GvjaHYRby3YC0CHyCBC/H2qfFaMveaw0yfezti+bQFNZYmINDcKO9IqJVXYe6dPhZGd7rGhrHjqCl6Z0I9LerTBz8fKte5dkqtScTWWYRhe1zz7+PRpF8Y4d9jxTGU5nQan8kvqpT8iIlJ3Vf/nrEgL1ts9stM5Ohh7gK/XtfAgP24ZmsAtQxMwDKPKJecenpqdkjInOUVlhAW6nldU6mCPuzi5T7sw4sMD6dPOztYjOdz07goycoooKHEwbXQiD17Wtb67KCIitaSRHWmVru4Vx8+Gd+J343tVe19NQQcgwNdGaIDrvwsqTmXtTM/F4TSICvYzNyQc3881SnTgeD4FJa6ztBbuzKxTH0REpH5oZEdaJT8fK78b37ventcm1J/cojKO5RbTLca1Q7OnXqd3uzAzNN0zvBNBfjbsgb742qz88qP17M7MrdUIkoiINAyFHZFaaBPiz/5j+V67KHvCTt925fVB/j427kzuBLimuawWyCoo5XheiTkdJiIijUvTWCK1YO61k1NkvubZObni0vaKAnxtdHBverjHvYmhiIg0PoUdkVowd1F2j+wUlznMoyZ6x1cedsC1+gtgt8KOiEiTUdgRqYWYUFcBsqdAeU9GHqUOg/AgX9pHBFb5vu7u+p49FY6UEBGRxqWwI1ILp5+PtSE1C4A+8WHVFh57dnnek6GwIyLSVBR2RGrh9LCzeJdrOXly16hq39c91jWy41mRJSIijU9hR6QWPLsoH88rpqjUwbK9roNEL+8ZU+37urYJ8VqRVRfFZQ5+2JbO4VMFdXq/iMj5TkvPRWrBM7JzIr+E5fuOU1jqIM4ecMYJ66fzrMg6eKKAPRm5dVp+/q/lB3nxu51YLHBJ9zZMvCCBq3rFYbNq3x4RkdrQyI5ILUQG+2G1gGHA/9YdBuDyxDa12ijwXFdkbT2SA7g+e/HuYzzwn/WMeWMJ87ZnaGpMRKQWFHZEasFmtRDlnsqauz0DgMtqmMLy6GHW7dStSDnVPX31f2NdZ2yFBfqyOyOP+z5cyy1/XVHpaewiIlJOYUeklmLcU1ClDgNfm4Xh3aJr9b7uMa6Rnb0VVmQ5nbUfkUk96Qo7F3WNZtroRJb86nIevKwr/j5W1hw8xYyFe2v9LBGR85HCjkgtVay3uaBzJCH+tSt5O31F1lcbj9Dn2R/4ZHVKje8tKCkzC5sT3LsxhwX6Mm10Ii9P6AfAhpRTZ9UPEZHzjcKOSC15VmRBzauwKqq4Imvx7mP86n+bKShx8PWmo173GYbBgeP5OCqM+qSeLARcAScs0Nfr/sEdIwDYnpZDUanjrPsjInK+UNgRqaWKIzuXJ9Y+7FQ8I+uB/6yjuMwJuA4SrVhgPHP9ES7/0yL+sXS/+ZpnCish8sxdmttHBBIV7Eepw2Db0Zyz64yIyHlEYUekljxhp0NkEF2ig8/qvZ4VWUWlThIiA/GzWckpKuPwqULznh+3pwOuFVceKe6w4wlLFVksFgYkhAOw0b2js4iInElhR6SWruoVS4/YEKZc0a1WS84r8qzI8vex8u4dg+kZ5wo/W4+4Tk43DIP1KVkA5gGjUL4SKyHizLADKOyIiNSCwo5ILbWPCOLHqZdyy5CEs37vhEHtGdYlkrcmDqR3fBh92tkB2HrUFXYOnyo0l5Afzysxv/dMY7WvZGQHYECHcAA2pqpIWUSkKtpBWaQRdGkTwif3J5s/944PA1LZ4t4wcP1pK6p2pufQJrSNWaBc2TQWQL/24YCrkPlEXrG5F5CIiJTTyI5IE+jTLgyAbe4i5Q3uKSyPnWmuZeqemp2EiDMLlMG1SqtrG1f9kKayREQqp7Aj0gQS40KxWS2cyC8hPafIHNnpHuOq7dmZnsuJ/BIKSx1YLNCuirADMCDBtQRdYUdEpHJNGnaWLFnC+PHjiY+Px2Kx8OWXX3pdNwyD3/72t7Rt25bAwEBGjhzJnj17vO45efIkkyZNwm63Ex4ezr333kteXt225RdpLAG+NjPYrD14iu3upeO3X9gBcE1jeUZ14uwB+PvYqnxWed1OVsM1WESkBWvSsJOfn0///v2ZMWNGpddfeeUV3nzzTd59911WrVpFcHAwo0aNoqioyLxn0qRJbNu2jblz5zJ79myWLFnC/fff31hdEKkzz1TWf1enUOY0iAn158rEWAD2ZORx4Fg+UL5zclUGVliRdTbHUIiInC+atEB5zJgxjBkzptJrhmHw+uuv8/TTT3PdddcB8OGHHxIbG8uXX37Jbbfdxo4dO/j+++9Zs2YNQ4YMAeCtt95i7Nix/OlPfyI+Pr7R+iJytvrE2/nfOli+7wQAgzpE0D4ikGA/G/klDpbsce23U9Wyc4+ecaEE+FrJLSpj//F8urlHjERExKXZ1uwcOHCA9PR0Ro4cab4WFhbGhRdeyIoVKwBYsWIF4eHhZtABGDlyJFarlVWrVlX57OLiYnJycry+RBqbZ2THY1DHcKxWi7kHz4KdmUDVK7E8fG1W+rqfVZ9TWUezCvn3ioOUuHd8FhFpqZpt2ElPd+0mGxsb6/V6bGyseS09PZ2YGO9t+318fIiMjDTvqcz06dMJCwszvxISzn7fFJFzldTWTsW9CQd1cBUaJ7Z17cGTW1QGVH5UxOkGuc/JWlJh9+Vz9dKcnTzz1TbeW7Kv3p4pItIUmm3YaUhPPfUU2dnZ5ldqampTN0nOQ8H+PnRt45py8rVZzJGeJPfIjkdNIzsAo3vHATB3ewYFJWX10r7Nh7MA+GL9Ea8zvEREWppmG3bi4ly/vDMyMrxez8jIMK/FxcWRmZnpdb2srIyTJ0+a91TG398fu93u9SXSFPrEu/7d6xUfRoCva8WVZ2THo6YCZXAdG9E+IpDCUoc5/XUuCkrKOOReDXbgeL55lIWISEvUbMNO586diYuLY/78+eZrOTk5rFq1iuRk1060ycnJZGVlsW7dOvOeBQsW4HQ6ufDCCxu9zSJny3N6+tW9yqdre1YY2fH3sdKmFrsiWywWxvd3FeR/s+lopfd8uiaFi6bPZ+3BkzU+b09GHhUHc2auP1zje0REmqsmDTt5eXls3LiRjRs3Aq6i5I0bN5KSkoLFYuHRRx/l97//PV9//TVbtmzhrrvuIj4+nuuvvx6ApKQkRo8ezS9+8QtWr17NsmXLmDJlCrfddptWYkmLcG3/eBY9cRkPXNrVfM0e4Eu7cFedTvuIQKzW2h06eq077CzcdYycolKva+tTTvGbWVs5ml3EP5cdqPFZnsNI7QGuBZvfbDpKcZmjVu0QEWlumjTsrF27loEDBzJw4EAAHnvsMQYOHMhvf/tbAH71q1/x0EMPcf/99zN06FDy8vL4/vvvCQgIMJ/x0UcfkZiYyJVXXsnYsWMZMWIE7733XpP0R+RsWSwWOkUHYzst0CS6R3dqU69T8T3dYkIoKXMyd1v59O+p/BIe+ngDZe49eBbuPEZhSfXBZac77Nw4qD1twwLIKSpjwY5znx4TEWkKTRp2LrvsMgzDOOPrgw8+AFx/ETz//POkp6dTVFTEvHnz6NGjh9czIiMj+fjjj8nNzSU7O5t//vOfhIRonxFp2TwHfJ7NnjkWi4Xx/VyjO1+7p7KcToPHPtvIkaxCOkUF0S7cVdezeHf1wWVXhms7hl5t7Vw/sB3gKlQWEWmJmm3Njsj57GcjOvH76/vw4GXdzup94/u3BWDp3uPMWLiXG95ZzsJdx/DzsTJj0iDG9nUV7s/ZWvXWDFA+jdUzLpQJg1xhZ9GuTE7kFZ9tV0REmpzCjkgzZA/w5Y5hHYkM9jur93VpE0KfdnYcToM//rCLTalZ2KwW/nB9H3rHhzG6jysMLdiRadbglDmc7EjLMZeXH8st5nheCRYL9IgNpVtMKP3bh1HmNPj1zC2UOirfZLCgpMw840tEpDlR2BFpZR68tBthgb6M6BbNC9f1Ztm0K7h5iGvjzIEJ4cTa/cktLmPZ3uOUOZzc9+FaxrzxE5+vc6248ozqdIwMItDPtRz+qbFJ+PlYmbs9g4f/u6HSwPObWVsZ++ZPLK5mY8OCkjIOHM+v7y6LiFRLYUeklRnXry2bfnc1/7nvQu5M7kRcWHlBv9VqMTcg/G5LOs9+s41Fu1zh5F/LDwKuE9fBewn8sC5RvHfnYPxsVuZsTefRTzZSViHwFJc5+N49NbZgh/feWBVN+2ILV/x5kU5oF5FGpbAjcp4Z09c1lTVrwxH+szIFiwV8rBa2Hc1h65HsCvU63psbXtYzhnfvHISvzcK3W9L437ryvXdWHzhJYalrWmz1wVOVfm5xmYO529MxDFi1/0RDdE1EpFIKOyLnmaGdIokO8cPhXor+f2OSzAD0yZoUdmW4ws7px1YAXJEYy6MjXSsiZ24oX521cGf51NXO9Jwz9vkB2JSaTVGpazRoT2ZePfVGRKRmCjsi5xmb1cJ1A1wrrCZd2IH7Lu7MbUNdNT1fbTjK7ozylViVuXFQOywW12jOkaxCABZVWMpuGLDu0JmjOyv2lY/mKOyISGNS2BE5Dz05qidfTR7O76/vg8ViIblLFAmRgeQWl1FU6iTA10rHqOBK39s2LJALOkUCrp2VD53IZ/+xfHysFkb1dh17UdmRFCv2Hze/35uRq8NFRaTRKOyInIcCfG30TwjHYnHt3Gy1WrhlcIJ5vXtM6Bm7Olfk2Wjwyw1HzALnwR0juMJ91tea0+p2ikodXoeJ5pc4SMsuqpe+iIjURGFHRAC4aUh7PPmmqiksjzF94vC1WdiZnmuu4ro8MYYh7hGfjalZXmdprU85RUmZkzah/nRt4xox0lSWiDQWhR0RAVzTU56Rmf4J4dXeGx7kx2U9Xffud++bc3nPGLpEBxMV7EdJmZOtR7LN+1e663Uu6hpFj1hXkNqrsCMijURhR0RMr9zUn5cn9DULlqtz3YB48/v4sAB6xIZgsVgY0ikC8J7KWuFeap7cJYru7vO+9mbmVvrcE3nFrKmk5kdEpK4UdkTEFBnsx61DO+Brq/lXw8ikWILdOyxf2jPGrP8Z6p7KWnPAFVgKSxzmJoLJXaPo5h7Z2ZNx5shOTlEpN/xlOTe/u0KBR0TqjcKOiNRJgK+NO4Z1xGa1cNPgdubrnrqdtYdO4XQarD10klKHQXxYAB0ig+jWxjWysyczz2tFlmEY/GbWVlJOFgAwr5qdmEVEzoZPUzdARFquaaMTefjK7gT7l/8q6R1vJ9DXRnZhKY9/volUd3gZ1jUKi8VClzbBWC2QXVjKsbxiYkJdx1l8vvYw32w6aj5n2d7jNKYftqVzNKuQey7qZI5SiUjroJEdEakzq9XiFXQAfG1WLujsGt2ZteEIa90bDI7oFg24RoQ6RAYBsNc9lbU3M5fffr0VgJ8P7wzAtqM5nMovMZ+bX1zG4VMFDdKP4jIHj3yygee+2e61RF5EWgeFHRGpdy9N6MvT45J47KoeTL68K0+NSWR8//KC5m4x7hVZx/JwOg0e/2wTRaVOLu4ezdPjkugRG4JhlBc2A9z7rzVc9sdFZ9TyvDV/D3f/czV5xWV1bu/WI+VHWWj6TKT1UdgRkXrXNiyQ+y7uwsNXdufJUYn8v0u7ehU9d4911+1k5DFrwxE2Hc4mxN+HP9/cH6vVwnD3KNBS91TWptQsVu4/SZnT4Pezt+N0n+u19uBJ/jx3N4t3H2POlrQ6t7fiyrH5CjsirY7Cjog0Os/y882Hs3jlh50ATL68GzF2V/3O8K6usLPcHXb+vfKQ+d5Nh7P5ZvNRHE6D3361zXz9XEZk1lYIO7sz8kg50TDTZSLSNBR2RKTRdXOHnU2Hs8nIKaZ9RCA/G97JvH5hl0hsVgsHTxSw9Ui2WbjsOXvrle938f6yA2xPy8HPx/VrbMnu4xSVlu/a/PaCPdz/4VqvnZwr43QarDvkmhqLDvEDNJUl0too7IhIo+vqXn7u8dSYJAJ8bebPoQG+9G8fBsATn2+iuMxJ73g7r986kDh7AEeyCvn9tzsAeHpcEnH2AApLHebJ6iknCvjz3N38uD2DVfur369n//E8ThWUEuBr5b6LuwAKOyKtjcKOiDS6YH8f2oUHAjCkYwRj+8adcY9n9dbOdNdOy3cO60ign40nRvU07+nV1s6kCztyZZLr6ApPSPlg+UE8W/jsSq98p2YPT73OgIRwRvd2tWP1gZPkFJXWtXsi0swo7IhIk7imf1vCg3x59trele5r4ylSBggN8OG6Aa6NC28Y2I4BCeH42iw8f11vbFYLI3u5prfm7cggp6iUz9ammu/dWWPYcY38DO0USafoYLrFhFDmNFjsPs1dRFo+bSooIk3iqTFJTBuViNVa+QZ+AztEEOhro7DUwc2DEwh0H01hs1r4+BcXkl1YStsw1+hQcpcogvxsZOQU87uvtpFXXIaP1UKZ02Bnek617fAUJ3t2fr4yKYa9mXnM25HhtVy+OnO3Z1BU6qj1/SLSuDSyIyJNpqqgA+DnY+WOYR1oF+5dvAwQ5OdjBh1wbVR4Sfc2gGsjQ4B7R7g2J9yTmUeZw1npZ2TmFJFysgCrBQZ1CAfgqiTXKNHCnZmUVvG+irIKSnjwP+t46L8bGmzTQxE5Nwo7ItJs/WZcL5b9+goS3DsuV8czlQUQFujLw1d2J8jPRkmZk4NVLCX37O6cGGcnNMAXcI0oRYf4kVNUxtztNRcqL9t7gjL3vj+eAmkRaV4UdkSkVbi8Zxs8pT+3X9iBYH8ferhPWK+qSLm8XifCfM1mtXD7BR0A+OfSAzV+7k97ymt7Vtaw8ktEmobCjoi0ClEh/tw8uD0dIoO456JOACTGucJOVXU7nrAz2F2v43HHsI742iysPXSKTalZVX6mYRj8tKf8wNKV+zWyI9IcKeyISKvxyk39WfKry4l178Tc0ww7Z47sbDuazdYjOdisFoZ19g47MfYAxvdzFRv/c1nVozv7j+dzJKsQP5sVH6uFI1mF5invItJ8KOyISKvlCTuVTWO9t2Q/ANf0a2seU1HRz9ynr3+7OY307KJKn//TbtcU1tDOEfRzb4K4QqM7Is2Owo6ItFqJcXYAUk4WkF/hVPTUkwXM3uw6OPT+S7pU+t6+7cO4oFMkZU6Df688WOk9nimsi7u3IblrFAArVaQs0uwo7IhIqxUZ7EdMqD8AuzLKR3f+sfQADqfBxd2j6R0fVuX7fz6iEwAfrUohq6DE61pJmdMcxbm4ezTDurjDzv4TGJ7tm0WkWVDYEZFW7fSprJP5JXyyJgWABy7tWu17r+oVR5foYLIKSnn00404neUhZn3KKQpKHESH+JEUZ2dwxwh8bRaOZheRerKw2ufuzshlb2beuXRLRM6Cwo6ItGqJp4Wdf684RFGp62DRi9xTT1WxWS28dftA/H2sLNp1jLcX7jWveZacj+gWjdVqIcjPh/7twwFYsf94ZY8DXJsQ3jBjGTfMWHbW52+VOpy8t2Qfc7akndX7RM53Cjsi0qr1dNft7EzP4fO1qcxwB5b/d2nXSs/kOl3v+DB+f30fAF6bt5vP1qTyyeoUvtnkChwj3Ds3A+V1O9Xst7N49zHySxzkFpfxw9b0Wvcjv7iMX3y4lhe/28nUzzZWuSu0iJxJYUdEWjXPyM7qAyd58n+bKXE4Gd07jnF929b6GTcPSWDiBQkYBvzqi838euYWUk4W4GO1cHH38gNLPXU7K/ZVXbezcGem+f3Xm47W6vOP5RZz23srWeQ+nLSo1MnBE/m1bn9lPlubyvqUU+f0DJGWQmFHRFq1bjEhWC3gKbd5+Mru/GXSIGzVnMtVmd+N783F3aOJDvFjRLdofnFxZz78+QXmnj4AgzpEEORnIz2niA+WHzzjGQ6nweLd5TsuL9t7nGO5xdV+bmGJg1v/uoItR7KJDPajXbjrTLAdadWf5l6ddYdO8qv/beaX/1mvYmo5LyjsiEirFuBrY0T3NgT72fjLpEE8dlWPag8gre45/773QtY+fRX/ue9CfjOuFxd1i/a6J9DPxrTRiQBMn7PzjJ2bN6ZmcaqgFHuAD33a2XEa8F0N9TffbUlj//F82oT688WDF3FJD9dn1nSae3XWH8oCID2niAPHz22ESKQlUNgRkVbvg3uGsvbpqxh7FlNXdXVXckeuSIyhpMzJI//dSFGpw7zmmcK6pEcbbhjYHqh5Ksuzcuzu5I50jg429w7aeQ4jO5sOZ5nfrzqg87yk9VPYEZFWz2q1EOhna5TPslgsvHJTP6JD/NmVkcv073aY1xbucoWdKxJjuKZfWywWWHfoVJVHTOzNzGPNwVNYLa66Iah43lfdw87mw9nm96sVduQ8oLAjIlLPokP8+dPN/QD414pDfLI6hYycIrYdzcFigUt7tCHWHsCwzq6C5m82Vz6686l7VOeKxBizNsgzsnMkq9Bcul5c5uCBf6/jpTk7a2zbqfwSUiqEq1XaBFHOAwo7IiIN4LKeMTx8ZXcAfvPlVl50j/D0bx9OVIhrV+drB7gOG/10TSq7M7xHakrKnHyx/ggAtw3tYL4eFuRLfJgr+Hj2Dlqy+zjfb0vn3cX7WHeo+hVWm4+4RnXahgXgY3Vtgnj4VPWbIIq0dAo7IiINZOrI7tw4sB0Op8FXG12jN5f3jDGvj+kTR3iQL4dOFDDq9SU8+skG9rhDz7wdGZzMLyHW7s9lPdt4PTexradux1WkPG97hnnttbm7q23TFne9ztBOkebhpbWp21m8+xiHT9XuRHen0+D7rekczVKIkuZBYUdEpIFYLBZemtCP5C7lOzVfkVgedsKD/PjiwYsY0ycOw4AvNx7lqteWcPmfFvHy964pqZsHJ+Bj8/5V7anb2ZGei9NpML/C3j1L9x5nVTUnr29y1+v0ax/GBe5ptOruB1iwM4O7/7maX360vjbd5p3F+3jgP+t47LONtbpfpKEp7IiINCA/Hyvv3jmYYV0iuSIxht7xdq/rXduE8M4dg5n90Aiu6hWLr83CgeP5HDrhGkW5dWjCGc/0jOzsSs9l85FsjucVE+Lvwy1DXCu8/jx3d5V1OJvdIzv92odzYedIAFYfrH5k58MVh9zvzSYjp6jae3ek5fD6PNfo0uoDJ8kuOLsjMUQagk9TN0BEpLULC/Tlk/uTq72nT7sw/nbXEHKLSvlpz3F+2nOMpLZ2EiKDzrg3qcJ5X3O3u46cuKRHNFOv6sGXG4+y+sBJlu09wYju3vsAZeQUkZFTjNUCfdrZKXMaWC1w6EQB6dlFxIUFnPFZqScLvDZCXLL7mLky7HQlZU4e/2wTpQ5X0HIarpGmcf0afsm/SHU0siMi0oyEBvgytm9bpt/Yj7uSO1V6T6foYPxsVvKKy/h0TSoAVybG0jYskNsvcBUzv/z9TkrKvM/P2pSaBUD3mFCC/HywB/jSyz3StOrACbILS/l601H2HSs/kf3TNalUHCRasqfqQ07fXriX7Wk5hAf5cr27+HrRrswq7xdpLAo7IiItjK/NSreYEACO55VgtcDl7lqgX17eldAAH7YcyWb6nB1e79tcoV7H40J33c4r3+9i6B/m8fB/N3DjX5azNzOPUoeTT9e6wtQ9F3UCYOmeYzicZ06R7c3MNQ9ZfeG6Ptw02DX6s3j3sWa1tL2wxMG/Vxwkq6CkqZsijahZh51nn30Wi8Xi9ZWYmGheLyoqYvLkyURFRRESEsKECRPIyMio5okiIq1DYttQ8/tBHSKIDPYDICY0gFdvGQDA+8sOeu3Q7Fl2XjHsXOCu2zmSVUhJmZMgPxvZhaX87IPVfLY2lWO5xUSH+DFtdCIh/j6cKihl29HyTQk9vtmUhsNpcEmPNozvH8+QThEE+trIzC0+pw0Q69u7i/fxzFfbeGP+nqZuijSiZh12AHr37k1aWpr5tXTpUvPa1KlT+eabb/j8889ZvHgxR48e5cYbb2zC1oqINI6kuPJC5yuTYr2uXdUrll9e1hWAX3+xmRX7TpB6ssBcdt6vfbh57xWJMdxzUSfuHdGZ2Q+N4KdfXU6HyCBSTxbym1lbAbhpcAKBfjYu6uoaBVpSoYbHY4F7Rdg17vqcAF8bye77F1dyf1Px7ENUcRdpaf2afdjx8fEhLi7O/IqOdhXcZWdn849//INXX32VK664gsGDB/P++++zfPlyVq5c2cStFhFpWBVHdkYmxZxx/bGrenBR1ygKShxM/NtKLn5lIacKSvG1Wbze62uz8uy1vXnmml70aRdGVIg/H/xsKOFBvuY9Ey9wTUld3MO138+S3d51O5k5RWxxjxpV3BPoUvf9zaVuxzAMczXarvTcZjW9Jg2r2YedPXv2EB8fT5cuXZg0aRIpKa7t09etW0dpaSkjR440701MTKRDhw6sWLGi2mcWFxeTk5Pj9SUi0pIMSAgn1u7PhZ0jzfqdinxsVt6cOJDh3aIID/Il0NeGzWrhpsEJ+PtUf05YlzYh/O2uIYQG+HBt/3g6RgUDcGl3V3hZn3KK3KLyJeWLdrlGbvq3DyMmtHxFlyf4rD14irziMq/PyC8u4x9LD9T69PYyh5NZGw5zMr/qWpuMnCKmz9nB/goF1hUdOlFATpGrHXnFZdo5+jzSrJeeX3jhhXzwwQf07NmTtLQ0nnvuOS6++GK2bt1Keno6fn5+hIeHe70nNjaW9PT0ap87ffp0nnvuuQZsuYhIwwoN8GXZtCsA1+aFlYkO8eej+4bV6flDO0Wy7umr8LWVP7tDVBCdooI4eKKA5ftOMKp3HFA+hXV5ovcIU8eo4PL79x7navf92QWl3PPBajakZBHga+X1Wwcwuk/1y9P/teIQL8zezhWJMfzznqFnXM8tKuXuf65mZ3ouu9Nzef9nF5xxT8XT3sF1mGplS/ul9WnWIztjxozh5ptvpl+/fowaNYrvvvuOrKwsPvvss3N67lNPPUV2drb5lZqaWk8tFhFpPD426xm7K9cnPx/rGUHqEnMqyzWaU1Lm5Kc9ru+vSDxzOs0zlfXXJftZse8EmblF3PreCjakZGGxQFGpkwf+s553F++rdlrJU2i9cFfmGafElzqc/PKj9WYh9E97jle6meHpdTq7ajmqJC1fsw47pwsPD6dHjx7s3buXuLg4SkpKyMrK8ronIyODuLi4ap/j7++P3W73+hIRkZp5zvb637rDbD2SzeoDJ8kvcRAd4k+f+LAz7r92QDtsVgvrDp1i4t9WMvylBexMzyU6xJ/ZD43g7uSOALw0ZyfXzVjGjIV72ZvpvXrr8KkCc48gw4CPV6eY1wzD4LdfbeWnPccJ9LXRNiyAMqfBD9vPHOHf4g47PWJd0347mtEqMWlYLSrs5OXlsW/fPtq2bcvgwYPx9fVl/vz55vVdu3aRkpJCcnL1O5WKiEjdXNqjDZf3bENxmZP/9+91zFx/GIArEttgtZ45nTa4YwRf/nI4Ey9IIMTfh1KHQbvwQD5/IJne8WE8d10ffje+Fz5WC5sPZ/PHH3Yx8tUlvDGvfGn491tdwSXU31V58dmaVIrLHAD8a/lB/rs6FYsF3pw4kInuTRW/3Zzm1Q6H02Cre8n8Le4doD0HqUrr16zDzhNPPMHixYs5ePAgy5cv54YbbsBmszFx4kTCwsK49957eeyxx1i4cCHr1q3jZz/7GcnJyQwbVrc5ahERqZ7VauH12wbSKSqII1mFzNxwBKh8Csujb/swpt/Yj9W/uZJ/3D2Er6cMp3N0sHn9Z8M7s/zXV/DiDX3NabK3F+4xp6vmuMPOIyO7E2cP4ER+Cd9vTWfz4Sz+8J1r48TfjE3iql6x5tEUy/Ye51SFYuZ9x/IoKHEQ5GdjbF/XPQeO51NU6qivPxppxpp12Dl8+DATJ06kZ8+e3HLLLURFRbFy5UratHH9n+G1117jmmuuYcKECVxyySXExcUxc+bMJm61iEjrFhboy1/vHEKQn2tVl6/NwojubWp4FwT5+XBlUixRIf5nXIuxB3D7hR348OcXMLxbFKUOgzfm7yEtu9DcG+eafvHc5l4G/8+lB5jy8QZKHQajesdy74jOgOtg1aS2rnO/fqwwleWZBuvTLoy2YQGEB/niNGBvZuUrt6R1adZh55NPPuHo0aMUFxdz+PBhPvnkE7p27WpeDwgIYMaMGZw8eZL8/HxmzpxZY72OiIicu55xofzp5v7YrBau6hVLiH/9Le59cpRrp/yZ6w/z9gLXERRDOkYQFxbAbUM7YLNa2HQ4m5STBbSPCOSVCf29Cqk9GxvOrjCV5dkHqF+7MNdu/O7DVJvT7s7ScJp12BERkeZrbN+2LP/1Fbx264B6fe6AhHCu7hWL04CPVrmKkce4p57iwgLMTRR9rBbemjiQsAobIHraBbB83wlzX55NnnPBEsIBSHTvQF1Z3U7KiQJmLNxb7Z4+0rIo7IiISJ3F2gNq3KSwLp4Y1ZOKq97H9CkftX/4yu50bRPMizf0ZWCHiDPe2zk6mN7xdhxOgw+WH6S4zMEOd6jp1861YqyqkZ21B09y3Yyl/PGHXUz7YnN9d0uaiMKOiIg0Oz1iQ7lhYDvANdITHx5oXusdH8b8xy/jlqEJVb7/xkHtAXhz/h6ueXMpJWVOwgJ96Rjl2kQwsa17ZKdC2Pl2cxq3/30Vp9x79MzdnmEuV2+OjmYVMnvzUcoczqZuSrOnsCMiIs3Sb8YmccewDjx7be+zfu89F3XiyVE9CfS1scddhNyvfZhZ29MjNgSLBY7nFZOZU8Tr83Yz+eP1lJQ5GZkUy9i+rpGk1+ftrvZz8orLeH3ebm56Z7l57lZdpWUXsubgyVqf2fXLj9Yz5eMN/N+sLTrnqwbN+rgIERE5f0WF+PP76/vW6b02q4XJl3fjxkHtePG7nXyz6ah5XAW4VoZ1jHQdZXHLX1dw8IRrmfs9F3XimWt6cehEPt9vTWf+zkw2pWbR313r41FU6uDjVSm8XaG259FPNvLdIxcT4Oua1kvPLuL7rWkkd42mZ1wolTmaVcjM9Yf5cXuGucPzLy7uzG/G9aq2fxtTs9joXmH22drDxIQG8MSonmf953S+UNgREZFWq21YIG9NHMgfb+pnhhCPnnGhHDxRwMETBQT52fjDDX24YaBr+qtLmxCuH9iOmeuP8Nq83XzgPmsrt6iU/6xM4R9LD3A8r9h1b3QwOUVl7D+ez4yFe3n86p4cyy3m1vdWcMgdohLjQrmmX1v6tAuje2woJWVO3l20j5kbDlPqcI3KWCyuHaL/9tMBYu0B3Hdxlyr79eHyg+Zn7z+ez9sL9xId4sc9wzvX659fa6GwIyIird7pQQdch53+sC2DxLhQZkwaRNc23qfHP3xFd77aeJRFu45x87vLySt2kHqywDzBPT4sgIev7M5Ng9vz4/YMfvnRet5ZtI/LE2P43VfbOHSigPAgXwqKHexMz61ymfuwLpFcP6AdVybF8sX6w7w0Zye//3YHMfYAru0ff8b9x/OKzWX1r906gCW7j/Hnubt5bvZ2ypwG947oXOXhsOcri6GJPnJycggLCyM7O1vnZImInCdKHU42pmbRt11YpWEIYNr/NvPpWu/DorvHhPDApV25dkA8vu6DWA3D4BcfrmPejgx8rBbKnAaRwX7874FkooL9+X5bGot3H2NPRh4HjudT5jS4IjGGyZd3Y3DH8hVlhmHw3Dfb+WD5QXxtFoZ3iya5SxQXd29Dr3jX308zFu7ljz/son9COF9NHo5hGDw/ezvvLzsIwC1D2vP76/vi51N/Zbn5xWXkFZcRaw+ot2fWh9r+/a2wg8KOiIhUrqCkjLnbM7BZLdgDfIkM9qNXW3ul54ClZRdy1atLyCsuI9DXxn/vH8aA02p9wBWyCoodZ+wP5OFwGjz22Ua+2njU6/WxfeP4v7FJ3PzuCtKyi3j1lv7mqjPDMPjnsoP84dvtOA24oFMk/7hnCKEB5Z9R6nByMr/kjMBy6EQ+ezLyuDIpptIRIafT4Ia/LGNnei5fTRlu7lEE8OWGI+zOyOWxq3rgY2v8NU8KO2dBYUdEROrDnC1p/GXRPp4c1dM856suDMNge1oOK/adYMW+EyzafQyH08DXZqHUYRAV7Mfyp644Y4+jRbsyeejjDeQWlzGqdyzv3jEYi8VCfnEZt/99FduOZPPFgxeZBddOp8GVry7mwPF8Xru1v1mzVNH3W9N44D/rAdfu1G/fPghwbb54xZ8XUeY0+NPN/blp8JnvbWi1/ftbS89FRETqyZi+bfnmoRHnFHQALBYLvePDuO/iLvzjnqHMfmgEgzqEm8XMEy/oUOlmjpf1jOHDey/A12bhh20Z/O2n/ZQ5nEz5eD2bUrMocxr8fekB8/6f9h7nwPF8AF6es4uCkjKv5xmGwVvuIzsAvt2Sxv5jrqX8r8/fTZnT1Z6/Lt6H01k+dnIst5id6TnNZkm8wo6IiEgzl9TWzv8euIhXJvTjzmEduf/SqldqDewQwW/Hu/Ymevn7Xdz7r7Us3HXMrOGZsyWNzJwiAD5aech8X3pOEX9dvN/rWYt2HWPb0RyC/Gxc0DkSw4C/Lt7PnoxcZrlPvA/wtbInM48FOzMBV9AZ9+ZPjH79J655aymfrUmlsKRpT5dX2BEREWkBrFYLtwxN4IXr+2APqLzex+OOCztw48B2OJwGi3cfw2KBtycOZEjHCMqcBh+vTiEtu5B5OzIAmDqyBwB/XbKPo1mFgGtU580Fe1zPG9aRaaPdB7RuOMzTX27FMGB07zjuTu4EwLvu0Z2pn24kM9e1LH/b0Rx+9cVmhk2fz8r9J+r9z6S2FHZERERaGYvFwh9u6EuS+1iMZ8f35urecdx1USfAdcDqv1ccchUzd47k4Su7cUGnSIpKnfzh2x3szcxjztZ0NqRk4e9j5b6LOzO4YwTDukRS6jBYdeAkFgs8dnUPfj6iM342K2sPnWLKf9ezdO9xAn1t/O+BZJ4ak0j7iECKyxzmeWRN8uehAmUVKIuISOtUVOogLbuIztHBAJSUORn+8gKO5RZjs1pwOA3enDiQa/vHs+VwNtfOWMrpqeCeizqZR3Ys3XOcO/6xCoAbBrYzT7w/fYl+xYJlh9Ngd0auGbzqkwqURUREznMBvjYz6AD4+Vi5/YIOgCuERAX7Map3LAB924fxyJXdibMHYA/wwc9mpV14IA9c2tV8//BuUVzcPZqIIF9z6gvg/ku7mKfU3ziwndfKLJvV0iBB52xoZAeN7IiIyPkjI6eI4S8toMxp8OBlXc1anNpyOA3KnM4zVoP9ZdFedqblMv3GvgT7N84BDbX9+1vHRYiIiJxHYu0BPHhZV+Zuz+Aedw3P2bBZLdisZy57/+Vl3eqhdQ1DIztoZEdERKQlUs2OiIiICAo7IiIi0sop7IiIiEirprAjIiIirZrCjoiIiLRqCjsiIiLSqinsiIiISKumsCMiIiKtmsKOiIiItGoKOyIiItKqKeyIiIhIq6awIyIiIq2awo6IiIi0ago7IiIi0qr5NHUDmgPDMADXUfEiIiLSMnj+3vb8PV4VhR0gNzcXgISEhCZuiYiIiJyt3NxcwsLCqrxuMWqKQ+cBp9PJ0aNHCQ0NxWKxNHVz6lVOTg4JCQmkpqZit9ubujkN7nzq7/nUV1B/W7vzqb/nU1+hYftrGAa5ubnEx8djtVZdmaORHcBqtdK+ffumbkaDstvt58X/qTzOp/6eT30F9be1O5/6ez71FRquv9WN6HioQFlERERaNYUdERERadUUdlo5f39/fve73+Hv79/UTWkU51N/z6e+gvrb2p1P/T2f+grNo78qUBYREZFWTSM7IiIi0qop7IiIiEirprAjIiIirZrCjoiIiLRqCjutwPTp0xk6dCihoaHExMRw/fXXs2vXLq97ioqKmDx5MlFRUYSEhDBhwgQyMjKaqMX166WXXsJisfDoo4+ar7W2/h45coQ77riDqKgoAgMD6du3L2vXrjWvG4bBb3/7W9q2bUtgYCAjR45kz549TdjiunE4HDzzzDN07tyZwMBAunbtygsvvOB17k1L7uuSJUsYP3488fHxWCwWvvzyS6/rtenbyZMnmTRpEna7nfDwcO69917y8vIasRe1V11/S0tLmTZtGn379iU4OJj4+Hjuuusujh496vWM1tLf0z3wwANYLBZef/11r9dbSn9r09cdO3Zw7bXXEhYWRnBwMEOHDiUlJcW83pi/pxV2WoHFixczefJkVq5cydy5cyktLeXqq68mPz/fvGfq1Kl88803fP755yxevJijR49y4403NmGr68eaNWv461//Sr9+/bxeb039PXXqFMOHD8fX15c5c+awfft2/vznPxMREWHe88orr/Dmm2/y7rvvsmrVKoKDgxk1ahRFRUVN2PKz9/LLL/POO+/w9ttvs2PHDl5++WVeeeUV3nrrLfOeltzX/Px8+vfvz4wZMyq9Xpu+TZo0iW3btjF37lxmz57NkiVLuP/++xurC2eluv4WFBSwfv16nnnmGdavX8/MmTPZtWsX1157rdd9raW/Fc2aNYuVK1cSHx9/xrWW0t+a+rpv3z5GjBhBYmIiixYtYvPmzTzzzDMEBASY9zTq72lDWp3MzEwDMBYvXmwYhmFkZWUZvr6+xueff27es2PHDgMwVqxY0VTNPGe5ublG9+7djblz5xqXXnqp8cgjjxiG0fr6O23aNGPEiBFVXnc6nUZcXJzxxz/+0XwtKyvL8Pf3N/773/82RhPrzbhx44yf//znXq/deOONxqRJkwzDaF19BYxZs2aZP9emb9u3bzcAY82aNeY9c+bMMSwWi3HkyJFGa3tdnN7fyqxevdoAjEOHDhmG0Tr7e/jwYaNdu3bG1q1bjY4dOxqvvfaaea2l9reyvt56663GHXfcUeV7Gvv3tEZ2WqHs7GwAIiMjAVi3bh2lpaWMHDnSvCcxMZEOHTqwYsWKJmljfZg8eTLjxo3z6he0vv5+/fXXDBkyhJtvvpmYmBgGDhzI3/72N/P6gQMHSE9P9+pvWFgYF154YYvr70UXXcT8+fPZvXs3AJs2bWLp0qWMGTMGaF19PV1t+rZixQrCw8MZMmSIec/IkSOxWq2sWrWq0dtc37Kzs7FYLISHhwOtr79Op5M777yTJ598kt69e59xvbX01+l08u2339KjRw9GjRpFTEwMF154oddUV2P/nlbYaWWcTiePPvoow4cPp0+fPgCkp6fj5+dn/gLxiI2NJT09vQlaee4++eQT1q9fz/Tp08+41tr6u3//ft555x26d+/ODz/8wIMPPsjDDz/Mv/71LwCzT7GxsV7va4n9/fWvf81tt91GYmIivr6+DBw4kEcffZRJkyYBrauvp6tN39LT04mJifG67uPjQ2RkZIvvf1FREdOmTWPixInmYZGtrb8vv/wyPj4+PPzww5Veby39zczMJC8vj5deeonRo0fz448/csMNN3DjjTeyePFioPF/T+vU81Zm8uTJbN26laVLlzZ1UxpMamoqjzzyCHPnzvWa/22tnE4nQ4YM4cUXXwRg4MCBbN26lXfffZe77767iVtXvz777DM++ugjPv74Y3r37s3GjRt59NFHiY+Pb3V9lXKlpaXccsstGIbBO++809TNaRDr1q3jjTfeYP369VgslqZuToNyOp0AXHfddUydOhWAAQMGsHz5ct59910uvfTSRm+TRnZakSlTpjB79mwWLlxI+/btzdfj4uIoKSkhKyvL6/6MjAzi4uIauZXnbt26dWRmZjJo0CB8fHzw8fFh8eLFvPnmm/j4+BAbG9uq+tu2bVt69erl9VpSUpK5qsHTp9NXMbTE/j755JPm6E7fvn258847mTp1qjmC15r6erra9C0uLo7MzEyv62VlZZw8ebLF9t8TdA4dOsTcuXPNUR1oXf396aefyMzMpEOHDubvrUOHDvH444/TqVMnoPX0Nzo6Gh8fnxp/bzXm72mFnVbAMAymTJnCrFmzWLBgAZ07d/a6PnjwYHx9fZk/f7752q5du0hJSSE5Obmxm3vOrrzySrZs2cLGjRvNryFDhjBp0iTz+9bU3+HDh5+xlcDu3bvp2LEjAJ07dyYuLs6rvzk5OaxatarF9begoACr1fvXks1mM/9LsTX19XS16VtycjJZWVmsW7fOvGfBggU4nU4uvPDCRm/zufIEnT179jBv3jyioqK8rrem/t55551s3rzZ6/dWfHw8Tz75JD/88APQevrr5+fH0KFDq/291eh/L9V7ybM0ugcffNAICwszFi1aZKSlpZlfBQUF5j0PPPCA0aFDB2PBggXG2rVrjeTkZCM5ObkJW12/Kq7GMozW1d/Vq1cbPj4+xh/+8Adjz549xkcffWQEBQUZ//nPf8x7XnrpJSM8PNz46quvjM2bNxvXXXed0blzZ6OwsLAJW3727r77bqNdu3bG7NmzjQMHDhgzZ840oqOjjV/96lfmPS25r7m5ucaGDRuMDRs2GIDx6quvGhs2bDBXH9Wmb6NHjzYGDhxorFq1yli6dKnRvXt3Y+LEiU3VpWpV19+SkhLj2muvNdq3b29s3LjR63dXcXGx+YzW0t/KnL4ayzBaTn9r6uvMmTMNX19f47333jP27NljvPXWW4bNZjN++ukn8xmN+XtaYacVACr9ev/99817CgsLjV/+8pdGRESEERQUZNxwww1GWlpa0zW6np0edlpbf7/55hujT58+hr+/v5GYmGi89957XtedTqfxzDPPGLGxsYa/v79x5ZVXGrt27Wqi1tZdTk6O8cgjjxgdOnQwAgICjC5duhi/+c1vvP7ya8l9XbhwYaX/X7377rsNw6hd306cOGFMnDjRCAkJMex2u/Gzn/3MyM3NbYLe1Ky6/h44cKDK310LFy40n9Fa+luZysJOS+lvbfr6j3/8w+jWrZsREBBg9O/f3/jyyy+9ntGYv6cthlFha1IRERGRVkY1OyIiItKqKeyIiIhIq6awIyIiIq2awo6IiIi0ago7IiIi0qop7IiIiEirprAjIiIirZrCjoiIiLRqCjsi0iQ6derE66+/Xuv7Fy1ahMViOePgQBGRmmgHZRGplcsuu4wBAwacVUCpzrFjxwgODiYoKKhW95eUlHDy5EliY2OxWCz10oaztWjRIi6//HJOnTpFeHh4k7RBRM6eT1M3QERaD8MwcDgc+PjU/KulTZs2Z/VsPz8/4uLi6to0ETmPaRpLRGp0zz33sHjxYt544w0sFgsWi4WDBw+aU0tz5sxh8ODB+Pv7s3TpUvbt28d1111HbGwsISEhDB06lHnz5nk98/RpLIvFwt///nduuOEGgoKC6N69O19//bV5/fRprA8++IDw8HB++OEHkpKSCAkJYfTo0aSlpZnvKSsr4+GHHyY8PJyoqCimTZvG3XffzfXXX19lXw8dOsT48eOJiIggODiY3r17891333Hw4EEuv/xyACIiIrBYLNxzzz0AOJ1Opk+fTufOnQkMDKR///7873//O6Pt3377Lf369SMgIIBhw4axdevWGj9XRM6dwo6I1OiNN94gOTmZX/ziF6SlpZGWlkZCQoJ5/de//jUvvfQSO3bsoF+/fuTl5TF27Fjmz5/Phg0bGD16NOPHjyclJaXaz3nuuee45ZZb2Lx5M2PHjmXSpEmcPHmyyvsLCgr405/+xL///W+WLFlCSkoKTzzxhHn95Zdf5qOPPuL9999n2bJl5OTk8OWXX1bbhsmTJ1NcXMySJUvYsmULL7/8MiEhISQkJPDFF18AsGvXLtLS0njjjTcAmD59Oh9++CHvvvsu27ZtY+rUqdxxxx0sXrzY69lPPvkkf/7zn1mzZg1t2rRh/PjxlJaWVvu5IlIPGuQsdRFpdS699FLjkUce8Xpt4cKFBmB8+eWXNb6/d+/exltvvWX+3LFjR+O1114zfwaMp59+2vw5Ly/PAIw5c+Z4fdapU6cMwzCM999/3wCMvXv3mu+ZMWOGERsba/4cGxtr/PGPfzR/LisrMzp06GBcd911Vbazb9++xrPPPlvptdPbYBiGUVRUZAQFBRnLly/3uvfee+81Jk6c6PW+Tz75xLx+4sQJIzAw0Pj0009r/FwROTeq2RGRczZkyBCvn/Py8nj22Wf59ttvSUtLo6ysjMLCwhpHdvr162d+HxwcjN1uJzMzs8r7g4KC6Nq1q/lz27Ztzfuzs7PJyMjgggsuMK/bbDYGDx6M0+ms8pkPP/wwDz74ID/++CMjR45kwoQJXu063d69eykoKOCqq67yer2kpISBAwd6vZacnGx+HxkZSc+ePdmxY0edPldEak/TWCJyzoKDg71+fuKJJ5g1axYvvvgiP/30Exs3bqRv376UlJRU+xxfX1+vny0WS7XBpLL7jXNcYHrfffexf/9+7rzzTrZs2cKQIUN46623qrw/Ly8PgG+//ZaNGzeaX9u3b/eq26nvzxWR2lPYEZFa8fPzw+Fw1OreZcuWcc8993DDDTfQt29f4uLiOHjwYMM28DRhYWHExsayZs0a8zWHw8H69etrfG9CQgIPPPAAM2fO5PHHH+dvf/sb4Poz8DzHo1evXvj7+5OSkkK3bt28virWNQGsXLnS/P7UqVPs3r2bpKSkGj9XRM6NprFEpFY6derEqlWrOHjwICEhIURGRlZ5b/fu3Zk5cybjx4/HYrHwzDPPVDtC01Aeeughpk+fTrdu3UhMTOStt97i1KlT1e7T8+ijjzJmzBh69OjBqVOnWLhwoRlIOnbsiMViYfbs2YwdO5bAwEBCQ0N54oknmDp1Kk6nkxEjRpCdnc2yZcuw2+3cfffd5rOff/55oqKiiI2N5Te/+Q3R0dHmyrDqPldEzo1GdkSkVp544glsNhu9evWiTZs21dbfvPrqq0RERHDRRRcxfvx4Ro0axaBBgxqxtS7Tpk1j4sSJ3HXXXSQnJxMSEsKoUaMICAio8j0Oh4PJkyeTlJTE6NGj6dGjB3/5y18AaNeuHc899xy//vWviY2NZcqUKQC88MILPPPMM0yfPt1837fffkvnzp29nv3SSy/xyCOPMHjwYNLT0/nmm2+8Rouq+lwROTfaQVlEzhtOp5OkpCRuueUWXnjhhUb7XO28LNK0NI0lIq3WoUOH+PHHH7n00kspLi7m7bff5sCBA9x+++1N3TQRaUSaxhKRVstqtfLBBx8wdOhQhg8fzpYtW5g3b55qYUTOM5rGEhERkVZNIzsiIiLSqinsiIiISKumsCMiIiKtmsKOiIiItGoKOyIiItKqKeyIiIhIq6awIyIiIq2awo6IiIi0av8fVHR/U/lmYbIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.6117125984251969, recall = 0.4445636623748212, f1 = 0.5149130074565038\n",
      "precision = 0.5422974176313446, recall = 0.37293325168401714, f1 = 0.441944847605225\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cbc927c-4dd1-4c50-8b13-ce127ed562dc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
